fix(search): observe label search rebuild failures

server/skillhub-app/src/main/java/com/iflytek/skillhub/metrics/SkillHubMetrics.java

@@ -60,4 +60,11 @@ public void incrementStorageAccessFailure(String operation) {
             "operation", operation
         ).increment();
     }
+
+    public void incrementSearchRebuildFailure(String trigger) {
+        meterRegistry.counter(
+            "skillhub.search.rebuild.failure",
+            "trigger", trigger
+        ).increment();
+    }
 }

server/skillhub-app/src/main/java/com/iflytek/skillhub/service/LabelSearchSyncService.java

@@ -1,5 +1,6 @@
 package com.iflytek.skillhub.service;
 
+import com.iflytek.skillhub.metrics.SkillHubMetrics;
 import com.iflytek.skillhub.search.SearchRebuildService;
 import java.util.List;
 import org.slf4j.Logger;
@@ -11,22 +12,21 @@
 public class LabelSearchSyncService {
 
     static final int REBUILD_BATCH_SIZE = 50;
+    static final int REBUILD_MAX_ATTEMPTS = 2;
 
     private static final Logger log = LoggerFactory.getLogger(LabelSearchSyncService.class);
 
     private final SearchRebuildService searchRebuildService;
+    private final SkillHubMetrics metrics;
 
-    public LabelSearchSyncService(SearchRebuildService searchRebuildService) {
+    public LabelSearchSyncService(SearchRebuildService searchRebuildService, SkillHubMetrics metrics) {
         this.searchRebuildService = searchRebuildService;
+        this.metrics = metrics;
     }
 
     @Async("skillhubEventExecutor")
     public void rebuildSkill(Long skillId) {
-        try {
-            searchRebuildService.rebuildBySkill(skillId);
-        } catch (RuntimeException ex) {
-            log.error("Failed to rebuild search document for skill {}", skillId, ex);
-        }
+        rebuildBySkill(skillId, "single", "Failed to rebuild search document for skill {}");
     }
 
     @Async("skillhubEventExecutor")
@@ -38,11 +38,7 @@ public void rebuildSkills(List<Long> skillIds) {
         for (int i = 0; i < normalizedSkillIds.size(); i += REBUILD_BATCH_SIZE) {
             List<Long> batch = normalizedSkillIds.subList(i, Math.min(i + REBUILD_BATCH_SIZE, normalizedSkillIds.size()));
             for (Long skillId : batch) {
-                try {
-                    searchRebuildService.rebuildBySkill(skillId);
-                } catch (RuntimeException ex) {
-                    log.error("Failed to rebuild search document for skill {} after label change", skillId, ex);
-                }
+                rebuildBySkill(skillId, "batch", "Failed to rebuild search document for skill {} after label change");
             }
         }
     }
@@ -51,4 +47,20 @@ public void rebuildSkills(List<Long> skillIds) {
     public void rebuildSkillsAsync(List<Long> skillIds) {
         rebuildSkills(skillIds);
     }
+
+    private void rebuildBySkill(Long skillId, String trigger, String finalFailureMessage) {
+        for (int attempt = 1; attempt <= REBUILD_MAX_ATTEMPTS; attempt++) {
+            try {
+                searchRebuildService.rebuildBySkill(skillId);
+                return;
+            } catch (RuntimeException ex) {
+                if (attempt < REBUILD_MAX_ATTEMPTS) {
+                    log.warn("Retrying search document rebuild for skill {} after attempt {}", skillId, attempt, ex);
+                    continue;
+                }
+                metrics.incrementSearchRebuildFailure(trigger);
+                log.error(finalFailureMessage, skillId, ex);
+            }
+        }
+    }
 }

server/skillhub-app/src/test/java/com/iflytek/skillhub/service/LabelSearchSyncIntegrationTest.java

@@ -2,6 +2,7 @@
 
 import com.iflytek.skillhub.SkillhubApplication;
 import com.iflytek.skillhub.TestRedisConfig;
+import com.iflytek.skillhub.auth.rbac.RbacService;
 import com.iflytek.skillhub.domain.label.LabelDefinition;
 import com.iflytek.skillhub.domain.label.LabelDefinitionRepository;
 import com.iflytek.skillhub.domain.label.LabelTranslation;
@@ -14,6 +15,8 @@
 import com.iflytek.skillhub.domain.skill.Skill;
 import com.iflytek.skillhub.domain.skill.SkillRepository;
 import com.iflytek.skillhub.domain.skill.SkillVisibility;
+import com.iflytek.skillhub.dto.AdminLabelUpdateRequest;
+import com.iflytek.skillhub.dto.LabelTranslationItemRequest;
 import com.iflytek.skillhub.infra.jpa.SkillSearchDocumentEntity;
 import com.iflytek.skillhub.infra.jpa.SkillSearchDocumentJpaRepository;
 import com.iflytek.skillhub.search.SearchEmbeddingService;
@@ -23,6 +26,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.UUID;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -58,6 +62,9 @@ class LabelSearchSyncIntegrationTest {
     @Autowired
     private SkillLabelAppService skillLabelAppService;
 
+    @Autowired
+    private LabelAdminAppService labelAdminAppService;
+
     @Autowired
     private NamespaceRepository namespaceRepository;
 
@@ -82,10 +89,14 @@ class LabelSearchSyncIntegrationTest {
     @MockBean
     private SearchEmbeddingService searchEmbeddingService;
 
+    @MockBean
+    private RbacService rbacService;
+
     @BeforeEach
     void setUp() {
         when(searchEmbeddingService.embed(anyString())).thenReturn("");
         when(searchEmbeddingService.similarity(anyString(), anyString())).thenReturn(0.0d);
+        when(rbacService.getUserRoleCodes(anyString())).thenReturn(Set.of("SUPER_ADMIN"));
     }
 
     @Test
@@ -193,6 +204,41 @@ public void afterCommit() {
                 .contains(f.labelDisplayName);
     }
 
+    @Test
+    void updatingLabelTranslations_rebuildsAffectedSkillSearchKeywords() throws Exception {
+        Fixture f = createFixture();
+        String updatedEnglish = "DeepLearning" + UUID.randomUUID().toString().substring(0, 8);
+        String updatedChinese = "深度学习" + UUID.randomUUID().toString().substring(0, 8);
+
+        skillLabelAppService.attachLabel(
+                f.namespaceSlug, f.skillSlug, f.labelSlug, f.ownerId, f.ownerRoles, auditContext());
+        SkillSearchDocumentEntity afterAttach = awaitIndexedDocument(f.skillId);
+        assertThat(afterAttach.getKeywords()).contains(f.labelDisplayName);
+
+        labelAdminAppService.update(
+                f.labelSlug,
+                new AdminLabelUpdateRequest(
+                        LabelType.RECOMMENDED,
+                        true,
+                        0,
+                        List.of(
+                                new LabelTranslationItemRequest("en", updatedEnglish),
+                                new LabelTranslationItemRequest("zh-CN", updatedChinese)
+                        )
+                ),
+                f.ownerId,
+                auditContext()
+        );
+
+        awaitKeywordPresent(f.skillId, updatedEnglish);
+        SkillSearchDocumentEntity afterUpdate = skillSearchDocumentJpaRepository.findBySkillId(f.skillId)
+                .orElseThrow();
+        assertThat(afterUpdate.getKeywords())
+                .contains(updatedEnglish)
+                .contains(updatedChinese)
+                .doesNotContain(f.labelDisplayName);
+    }
+
     private SkillSearchDocumentEntity awaitIndexedDocument(Long skillId) throws InterruptedException {
         Instant deadline = Instant.now().plus(Duration.ofSeconds(15));
         Optional<SkillSearchDocumentEntity> indexed = skillSearchDocumentJpaRepository.findBySkillId(skillId);
@@ -222,6 +268,24 @@ private void awaitKeywordAbsent(Long skillId, String keyword) throws Interrupted
                         + skillId + " but keywords were: " + keywords);
     }
 
+    private void awaitKeywordPresent(Long skillId, String keyword) throws InterruptedException {
+        Instant deadline = Instant.now().plus(Duration.ofSeconds(15));
+        while (Instant.now().isBefore(deadline)) {
+            Optional<SkillSearchDocumentEntity> indexed =
+                    skillSearchDocumentJpaRepository.findBySkillId(skillId);
+            if (indexed.isPresent() && indexed.get().getKeywords().contains(keyword)) {
+                return;
+            }
+            Thread.sleep(100L);
+        }
+        String keywords = skillSearchDocumentJpaRepository.findBySkillId(skillId)
+                .map(SkillSearchDocumentEntity::getKeywords)
+                .orElse("<no document>");
+        throw new AssertionError(
+                "Expected keyword '" + keyword + "' to be present in index for skill "
+                        + skillId + " but keywords were: " + keywords);
+    }
+
     private AuditRequestContext auditContext() {
         return new AuditRequestContext("127.0.0.1", "junit");
     }

server/skillhub-app/src/test/java/com/iflytek/skillhub/service/LabelSearchSyncServiceTest.java

@@ -1,12 +1,18 @@
 package com.iflytek.skillhub.service;
 
+import com.iflytek.skillhub.metrics.SkillHubMetrics;
 import com.iflytek.skillhub.search.SearchRebuildService;
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
 import org.junit.jupiter.api.Test;
 
 import java.util.ArrayList;
 import java.util.List;
 
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 
@@ -15,7 +21,11 @@ class LabelSearchSyncServiceTest {
     @Test
     void rebuildSkillsShouldSkipNullsAndDuplicatesWhileProcessingLargeLists() {
         SearchRebuildService rebuildService = mock(SearchRebuildService.class);
-        LabelSearchSyncService service = new LabelSearchSyncService(rebuildService);
+        SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry();
+        LabelSearchSyncService service = new LabelSearchSyncService(
+            rebuildService,
+            new SkillHubMetrics(meterRegistry)
+        );
         List<Long> skillIds = new ArrayList<>();
         skillIds.add(null);
         for (long i = 1; i <= 120; i++) {
@@ -30,5 +40,69 @@ void rebuildSkillsShouldSkipNullsAndDuplicatesWhileProcessingLargeLists() {
             verify(rebuildService).rebuildBySkill(i);
         }
         verifyNoMoreInteractions(rebuildService);
+        assertThat(meterRegistry.find("skillhub.search.rebuild.failure").counter()).isNull();
+    }
+
+    @Test
+    void rebuildSkillShouldRetryTransientFailureWithoutRecordingFailureMetric() {
+        SearchRebuildService rebuildService = mock(SearchRebuildService.class);
+        SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry();
+        LabelSearchSyncService service = new LabelSearchSyncService(
+            rebuildService,
+            new SkillHubMetrics(meterRegistry)
+        );
+        doThrow(new IllegalStateException("temporary index failure"))
+            .doNothing()
+            .when(rebuildService)
+            .rebuildBySkill(42L);
+
+        service.rebuildSkill(42L);
+
+        verify(rebuildService, times(2)).rebuildBySkill(42L);
+        assertThat(meterRegistry.find("skillhub.search.rebuild.failure").counter()).isNull();
+    }
+
+    @Test
+    void rebuildSkillShouldRecordFailureMetricAfterRetriesAreExhausted() {
+        SearchRebuildService rebuildService = mock(SearchRebuildService.class);
+        SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry();
+        LabelSearchSyncService service = new LabelSearchSyncService(
+            rebuildService,
+            new SkillHubMetrics(meterRegistry)
+        );
+        doThrow(new IllegalStateException("index unavailable"))
+            .when(rebuildService)
+            .rebuildBySkill(42L);
+
+        service.rebuildSkill(42L);
+
+        verify(rebuildService, times(LabelSearchSyncService.REBUILD_MAX_ATTEMPTS)).rebuildBySkill(42L);
+        assertThat(meterRegistry.get("skillhub.search.rebuild.failure")
+            .tag("trigger", "single")
+            .counter()
+            .count()).isEqualTo(1.0d);
+    }
+
+    @Test
+    void rebuildSkillsShouldRecordBatchFailureMetricAndContinue() {
+        SearchRebuildService rebuildService = mock(SearchRebuildService.class);
+        SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry();
+        LabelSearchSyncService service = new LabelSearchSyncService(
+            rebuildService,
+            new SkillHubMetrics(meterRegistry)
+        );
+        doThrow(new IllegalStateException("temporary index failure"))
+            .when(rebuildService)
+            .rebuildBySkill(2L);
+
+        service.rebuildSkills(List.of(1L, 2L, 3L));
+
+        verify(rebuildService).rebuildBySkill(1L);
+        verify(rebuildService, times(LabelSearchSyncService.REBUILD_MAX_ATTEMPTS)).rebuildBySkill(2L);
+        verify(rebuildService).rebuildBySkill(3L);
+        assertThat(meterRegistry.get("skillhub.search.rebuild.failure")
+            .tag("trigger", "batch")
+            .counter()
+            .count()).isEqualTo(1.0d);
     }
 }