From b6cd4145884fa9536030a06b09a610240fc7cfa8 Mon Sep 17 00:00:00 2001 From: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com> Date: Tue, 4 Aug 2026 10:43:24 +0800 Subject: [PATCH] fix(search): observe label search rebuild failures Signed-off-by: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com> --- .../skillhub/metrics/SkillHubMetrics.java | 7 ++ .../service/LabelSearchSyncService.java | 34 ++++++--- .../LabelSearchSyncIntegrationTest.java | 64 ++++++++++++++++ .../service/LabelSearchSyncServiceTest.java | 76 ++++++++++++++++++- 4 files changed, 169 insertions(+), 12 deletions(-) diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/metrics/SkillHubMetrics.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/metrics/SkillHubMetrics.java index abbd05dd3..ac2055319 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/metrics/SkillHubMetrics.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/metrics/SkillHubMetrics.java @@ -60,4 +60,11 @@ public void incrementStorageAccessFailure(String operation) { "operation", operation ).increment(); } + + public void incrementSearchRebuildFailure(String trigger) { + meterRegistry.counter( + "skillhub.search.rebuild.failure", + "trigger", trigger + ).increment(); + } } diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/LabelSearchSyncService.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/LabelSearchSyncService.java index 0dc9ce412..da5d8300d 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/LabelSearchSyncService.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/service/LabelSearchSyncService.java @@ -1,5 +1,6 @@ package com.iflytek.skillhub.service; +import com.iflytek.skillhub.metrics.SkillHubMetrics; import com.iflytek.skillhub.search.SearchRebuildService; import java.util.List; import org.slf4j.Logger; @@ -11,22 +12,21 @@ public class LabelSearchSyncService { static final int REBUILD_BATCH_SIZE = 50; + static final int REBUILD_MAX_ATTEMPTS = 2; private static final Logger log = LoggerFactory.getLogger(LabelSearchSyncService.class); private final SearchRebuildService searchRebuildService; + private final SkillHubMetrics metrics; - public LabelSearchSyncService(SearchRebuildService searchRebuildService) { + public LabelSearchSyncService(SearchRebuildService searchRebuildService, SkillHubMetrics metrics) { this.searchRebuildService = searchRebuildService; + this.metrics = metrics; } @Async("skillhubEventExecutor") public void rebuildSkill(Long skillId) { - try { - searchRebuildService.rebuildBySkill(skillId); - } catch (RuntimeException ex) { - log.error("Failed to rebuild search document for skill {}", skillId, ex); - } + rebuildBySkill(skillId, "single", "Failed to rebuild search document for skill {}"); } @Async("skillhubEventExecutor") @@ -38,11 +38,7 @@ public void rebuildSkills(List skillIds) { for (int i = 0; i < normalizedSkillIds.size(); i += REBUILD_BATCH_SIZE) { List batch = normalizedSkillIds.subList(i, Math.min(i + REBUILD_BATCH_SIZE, normalizedSkillIds.size())); for (Long skillId : batch) { - try { - searchRebuildService.rebuildBySkill(skillId); - } catch (RuntimeException ex) { - log.error("Failed to rebuild search document for skill {} after label change", skillId, ex); - } + rebuildBySkill(skillId, "batch", "Failed to rebuild search document for skill {} after label change"); } } } @@ -51,4 +47,20 @@ public void rebuildSkills(List skillIds) { public void rebuildSkillsAsync(List skillIds) { rebuildSkills(skillIds); } + + private void rebuildBySkill(Long skillId, String trigger, String finalFailureMessage) { + for (int attempt = 1; attempt <= REBUILD_MAX_ATTEMPTS; attempt++) { + try { + searchRebuildService.rebuildBySkill(skillId); + return; + } catch (RuntimeException ex) { + if (attempt < REBUILD_MAX_ATTEMPTS) { + log.warn("Retrying search document rebuild for skill {} after attempt {}", skillId, attempt, ex); + continue; + } + metrics.incrementSearchRebuildFailure(trigger); + log.error(finalFailureMessage, skillId, ex); + } + } + } } diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/LabelSearchSyncIntegrationTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/LabelSearchSyncIntegrationTest.java index 257d33d28..45719e318 100644 --- a/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/LabelSearchSyncIntegrationTest.java +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/LabelSearchSyncIntegrationTest.java @@ -2,6 +2,7 @@ import com.iflytek.skillhub.SkillhubApplication; import com.iflytek.skillhub.TestRedisConfig; +import com.iflytek.skillhub.auth.rbac.RbacService; import com.iflytek.skillhub.domain.label.LabelDefinition; import com.iflytek.skillhub.domain.label.LabelDefinitionRepository; import com.iflytek.skillhub.domain.label.LabelTranslation; @@ -14,6 +15,8 @@ import com.iflytek.skillhub.domain.skill.Skill; import com.iflytek.skillhub.domain.skill.SkillRepository; import com.iflytek.skillhub.domain.skill.SkillVisibility; +import com.iflytek.skillhub.dto.AdminLabelUpdateRequest; +import com.iflytek.skillhub.dto.LabelTranslationItemRequest; import com.iflytek.skillhub.infra.jpa.SkillSearchDocumentEntity; import com.iflytek.skillhub.infra.jpa.SkillSearchDocumentJpaRepository; import com.iflytek.skillhub.search.SearchEmbeddingService; @@ -23,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.UUID; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -58,6 +62,9 @@ class LabelSearchSyncIntegrationTest { @Autowired private SkillLabelAppService skillLabelAppService; + @Autowired + private LabelAdminAppService labelAdminAppService; + @Autowired private NamespaceRepository namespaceRepository; @@ -82,10 +89,14 @@ class LabelSearchSyncIntegrationTest { @MockBean private SearchEmbeddingService searchEmbeddingService; + @MockBean + private RbacService rbacService; + @BeforeEach void setUp() { when(searchEmbeddingService.embed(anyString())).thenReturn(""); when(searchEmbeddingService.similarity(anyString(), anyString())).thenReturn(0.0d); + when(rbacService.getUserRoleCodes(anyString())).thenReturn(Set.of("SUPER_ADMIN")); } @Test @@ -193,6 +204,41 @@ public void afterCommit() { .contains(f.labelDisplayName); } + @Test + void updatingLabelTranslations_rebuildsAffectedSkillSearchKeywords() throws Exception { + Fixture f = createFixture(); + String updatedEnglish = "DeepLearning" + UUID.randomUUID().toString().substring(0, 8); + String updatedChinese = "深度学习" + UUID.randomUUID().toString().substring(0, 8); + + skillLabelAppService.attachLabel( + f.namespaceSlug, f.skillSlug, f.labelSlug, f.ownerId, f.ownerRoles, auditContext()); + SkillSearchDocumentEntity afterAttach = awaitIndexedDocument(f.skillId); + assertThat(afterAttach.getKeywords()).contains(f.labelDisplayName); + + labelAdminAppService.update( + f.labelSlug, + new AdminLabelUpdateRequest( + LabelType.RECOMMENDED, + true, + 0, + List.of( + new LabelTranslationItemRequest("en", updatedEnglish), + new LabelTranslationItemRequest("zh-CN", updatedChinese) + ) + ), + f.ownerId, + auditContext() + ); + + awaitKeywordPresent(f.skillId, updatedEnglish); + SkillSearchDocumentEntity afterUpdate = skillSearchDocumentJpaRepository.findBySkillId(f.skillId) + .orElseThrow(); + assertThat(afterUpdate.getKeywords()) + .contains(updatedEnglish) + .contains(updatedChinese) + .doesNotContain(f.labelDisplayName); + } + private SkillSearchDocumentEntity awaitIndexedDocument(Long skillId) throws InterruptedException { Instant deadline = Instant.now().plus(Duration.ofSeconds(15)); Optional indexed = skillSearchDocumentJpaRepository.findBySkillId(skillId); @@ -222,6 +268,24 @@ private void awaitKeywordAbsent(Long skillId, String keyword) throws Interrupted + skillId + " but keywords were: " + keywords); } + private void awaitKeywordPresent(Long skillId, String keyword) throws InterruptedException { + Instant deadline = Instant.now().plus(Duration.ofSeconds(15)); + while (Instant.now().isBefore(deadline)) { + Optional indexed = + skillSearchDocumentJpaRepository.findBySkillId(skillId); + if (indexed.isPresent() && indexed.get().getKeywords().contains(keyword)) { + return; + } + Thread.sleep(100L); + } + String keywords = skillSearchDocumentJpaRepository.findBySkillId(skillId) + .map(SkillSearchDocumentEntity::getKeywords) + .orElse(""); + throw new AssertionError( + "Expected keyword '" + keyword + "' to be present in index for skill " + + skillId + " but keywords were: " + keywords); + } + private AuditRequestContext auditContext() { return new AuditRequestContext("127.0.0.1", "junit"); } diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/LabelSearchSyncServiceTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/LabelSearchSyncServiceTest.java index c6859bede..00dc22076 100644 --- a/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/LabelSearchSyncServiceTest.java +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/service/LabelSearchSyncServiceTest.java @@ -1,12 +1,18 @@ package com.iflytek.skillhub.service; +import com.iflytek.skillhub.metrics.SkillHubMetrics; import com.iflytek.skillhub.search.SearchRebuildService; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.List; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; @@ -15,7 +21,11 @@ class LabelSearchSyncServiceTest { @Test void rebuildSkillsShouldSkipNullsAndDuplicatesWhileProcessingLargeLists() { SearchRebuildService rebuildService = mock(SearchRebuildService.class); - LabelSearchSyncService service = new LabelSearchSyncService(rebuildService); + SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry(); + LabelSearchSyncService service = new LabelSearchSyncService( + rebuildService, + new SkillHubMetrics(meterRegistry) + ); List skillIds = new ArrayList<>(); skillIds.add(null); for (long i = 1; i <= 120; i++) { @@ -30,5 +40,69 @@ void rebuildSkillsShouldSkipNullsAndDuplicatesWhileProcessingLargeLists() { verify(rebuildService).rebuildBySkill(i); } verifyNoMoreInteractions(rebuildService); + assertThat(meterRegistry.find("skillhub.search.rebuild.failure").counter()).isNull(); + } + + @Test + void rebuildSkillShouldRetryTransientFailureWithoutRecordingFailureMetric() { + SearchRebuildService rebuildService = mock(SearchRebuildService.class); + SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry(); + LabelSearchSyncService service = new LabelSearchSyncService( + rebuildService, + new SkillHubMetrics(meterRegistry) + ); + doThrow(new IllegalStateException("temporary index failure")) + .doNothing() + .when(rebuildService) + .rebuildBySkill(42L); + + service.rebuildSkill(42L); + + verify(rebuildService, times(2)).rebuildBySkill(42L); + assertThat(meterRegistry.find("skillhub.search.rebuild.failure").counter()).isNull(); + } + + @Test + void rebuildSkillShouldRecordFailureMetricAfterRetriesAreExhausted() { + SearchRebuildService rebuildService = mock(SearchRebuildService.class); + SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry(); + LabelSearchSyncService service = new LabelSearchSyncService( + rebuildService, + new SkillHubMetrics(meterRegistry) + ); + doThrow(new IllegalStateException("index unavailable")) + .when(rebuildService) + .rebuildBySkill(42L); + + service.rebuildSkill(42L); + + verify(rebuildService, times(LabelSearchSyncService.REBUILD_MAX_ATTEMPTS)).rebuildBySkill(42L); + assertThat(meterRegistry.get("skillhub.search.rebuild.failure") + .tag("trigger", "single") + .counter() + .count()).isEqualTo(1.0d); + } + + @Test + void rebuildSkillsShouldRecordBatchFailureMetricAndContinue() { + SearchRebuildService rebuildService = mock(SearchRebuildService.class); + SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry(); + LabelSearchSyncService service = new LabelSearchSyncService( + rebuildService, + new SkillHubMetrics(meterRegistry) + ); + doThrow(new IllegalStateException("temporary index failure")) + .when(rebuildService) + .rebuildBySkill(2L); + + service.rebuildSkills(List.of(1L, 2L, 3L)); + + verify(rebuildService).rebuildBySkill(1L); + verify(rebuildService, times(LabelSearchSyncService.REBUILD_MAX_ATTEMPTS)).rebuildBySkill(2L); + verify(rebuildService).rebuildBySkill(3L); + assertThat(meterRegistry.get("skillhub.search.rebuild.failure") + .tag("trigger", "batch") + .counter() + .count()).isEqualTo(1.0d); } }