I18N-2066 Rewrite Processor Implementation

webapp/pom.xml

@@ -316,6 +316,11 @@
             <artifactId>opennlp-tools</artifactId>
             <version>2.1.0</version>
         </dependency>
+        <dependency>
+            <groupId>org.ahocorasick</groupId>
+            <artifactId>ahocorasick</artifactId>
+            <version>0.6.3</version>
+        </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>auth</artifactId>

webapp/src/main/java/com/box/l10n/mojito/service/rewriterule/RewriteRuleProcessor.java

@@ -0,0 +1,112 @@
+package com.box.l10n.mojito.service.rewriterule;
+
+import com.box.l10n.mojito.entity.RewriteRule;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import org.ahocorasick.trie.Emit;
+import org.ahocorasick.trie.Trie;
+import org.springframework.stereotype.Service;
+import org.springframework.util.Assert;
+import org.springframework.util.StringUtils;
+
+@Service
+public class RewriteRuleProcessor {
+
+  private final RewriteRuleService rewriteRuleService;
+
+  public RewriteRuleProcessor(RewriteRuleService rewriteRuleService) {
+    this.rewriteRuleService = rewriteRuleService;
+  }
+
+  private String resolveVariables(String rewriteTo, String localeTag) {
+    Assert.notNull(rewriteTo, "rewriteTo must not be null");
+    Assert.notNull(localeTag, "localeTag must not be null");
+
+    Locale locale = Locale.forLanguageTag(localeTag);
+    String languageValue = locale.getLanguage() == null ? "" : locale.getLanguage();
+    String countryValue = locale.getCountry() == null ? "" : locale.getCountry();
+
+    return rewriteTo
+        .replace("$language", languageValue)
+        .replace("$country", countryValue)
+        .replace("$locale", localeTag);
+  }
+
+  private boolean isRepositorySpecific(RewriteRule rewriteRule) {
+    return rewriteRule.getRepository() != null;
+  }
+
+  private int getRulePriority(RewriteRule rewriteRule) {
+    // Rewrite rules currently only model scope-based precedence.
+    return this.isRepositorySpecific(rewriteRule) ? 1 : 0;
+  }
+
+  public String processExactMatches(
+      String content, Long repositoryId, Long localeId, String localeTag) {
+    Assert.notNull(repositoryId, "repositoryId must not be null");
+    Assert.notNull(localeId, "localeId must not be null");
+    Assert.notNull(localeTag, "localeTag must not be null");
+
+    if (!StringUtils.hasLength(content)) {
+      return content;
+    }
+
+    List<RewriteRule> rewriteRules =
+        rewriteRuleService.findActiveRewriteRules(localeId, repositoryId).stream()
+            .filter(rewriteRule -> StringUtils.hasLength(rewriteRule.getRewriteFrom()))
+            .toList();
+
+    if (rewriteRules.isEmpty()) {
+      return content;
+    }
+
+    Map<String, List<RewriteRule>> rulesByRewriteFrom = new HashMap<>();
+    Trie.TrieBuilder trieBuilder = Trie.builder().ignoreOverlaps();
+
+    rewriteRules.forEach(
+        rewriteRule -> {
+          trieBuilder.addKeyword(rewriteRule.getRewriteFrom());
+          rulesByRewriteFrom
+              .computeIfAbsent(rewriteRule.getRewriteFrom(), key -> new ArrayList<>())
+              .add(rewriteRule);
+        });
+
+    Trie trie = trieBuilder.build();
+    Collection<Emit> emits = trie.parseText(content);
+
+    if (emits.isEmpty()) {
+      return content;
+    }
+
+    List<Match> matches = new ArrayList<>();
+    for (Emit emit : emits) {
+      List<RewriteRule> emitRules = rulesByRewriteFrom.get(emit.getKeyword());
+      emitRules.sort(
+          (leftRule, rightRule) ->
+              Integer.compare(this.getRulePriority(rightRule), this.getRulePriority(leftRule)));
+      RewriteRule emitRule = emitRules.getFirst();
+      matches.add(new Match(emit.getStart(), emit.getEnd(), emitRule));
+    }
+
+    matches.sort(Comparator.comparingInt(Match::start));
+
+    StringBuilder rewrittenContent = new StringBuilder();
+    int currentIndex = 0;
+
+    for (Match selectedCandidate : matches) {
+      rewrittenContent.append(content, currentIndex, selectedCandidate.start());
+      rewrittenContent.append(resolveVariables(selectedCandidate.rule().getRewriteTo(), localeTag));
+      currentIndex = selectedCandidate.end() + 1;
+    }
+
+    rewrittenContent.append(content, currentIndex, content.length());
+    return rewrittenContent.toString();
+  }
+
+  private record Match(int start, int end, RewriteRule rule) {}
+}

webapp/src/main/java/com/box/l10n/mojito/service/rewriterule/RewriteRuleService.java

@@ -17,12 +17,14 @@
 import com.box.l10n.mojito.service.repository.RepositoryRepository;
 import jakarta.persistence.criteria.Predicate;
 import java.util.ArrayList;
+import java.util.List;
 import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.jpa.domain.Specification;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
+import org.springframework.util.Assert;
 
 @Service
 public class RewriteRuleService {
@@ -83,6 +85,29 @@ public Page<RewriteRule> findRewriteRules(
     return rewriteRuleRepository.findAll(spec, pageable);
   }
 
+  public List<RewriteRule> findActiveRewriteRules(Long localeId, Long repositoryId) {
+
+    Assert.notNull(repositoryId, "repositoryId must not be null");
+    Assert.notNull(localeId, "localeId must not be null");
+
+    Specification<RewriteRule> spec =
+        (root, query, builder) -> {
+          var predicates = new ArrayList<Predicate>();
+
+          predicates.add(builder.equal(root.get("enabled"), true));
+          predicates.add(builder.equal(root.get("locale").get("id"), localeId));
+
+          predicates.add(
+              builder.or(
+                  builder.isNull(root.get("repository")),
+                  builder.equal(root.get("repository").get("id"), repositoryId)));
+
+          return builder.and(predicates.toArray(new Predicate[0]));
+        };
+
+    return rewriteRuleRepository.findAll(spec);
+  }
+
   public RewriteRule getRewriteRuleById(Long id) throws RewriteRuleWithIdNotFoundException {
     return rewriteRuleRepository
         .findById(id)

webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java

@@ -5,12 +5,14 @@
 import static com.box.l10n.mojito.service.thirdparty.ThirdPartyTMSUtils.isFileEqualToPreviousRun;
 import static com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFileUtils.isPluralFile;
 
+import com.box.l10n.mojito.entity.Locale;
 import com.box.l10n.mojito.entity.Repository;
 import com.box.l10n.mojito.entity.RepositoryLocale;
 import com.box.l10n.mojito.iterators.PageFetcherOffsetAndLimitSplitIterator;
 import com.box.l10n.mojito.iterators.Spliterators;
 import com.box.l10n.mojito.service.ai.translation.AITranslationConfiguration;
 import com.box.l10n.mojito.service.ai.translation.AITranslationService;
+import com.box.l10n.mojito.service.rewriterule.RewriteRuleProcessor;
 import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingJsonConverter;
 import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingOptions;
 import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingParsedFileResponse;
@@ -72,6 +74,8 @@ public class ThirdPartyTMSSmartlingWithJson {
 
   AITranslationService aiTranslationService;
 
+  RewriteRuleProcessor rewriteRuleProcessor;
+
   int batchSize = 5000;
 
   public ThirdPartyTMSSmartlingWithJson(
@@ -83,6 +87,7 @@ public ThirdPartyTMSSmartlingWithJson(
       MeterRegistry meterRegistry,
       ThirdPartyFileChecksumRepository thirdPartyFileChecksumRepository,
       AITranslationConfiguration aiTranslationConfiguration,
+      RewriteRuleProcessor rewriteRuleProcessor,
       @Autowired(required = false) AITranslationService aiTranslationService) {
     this.smartlingClient = smartlingClient;
     this.smartlingJsonConverter = smartlingJsonConverter;
@@ -92,6 +97,7 @@ public ThirdPartyTMSSmartlingWithJson(
     this.meterRegistry = meterRegistry;
     this.thirdPartyFileChecksumRepository = thirdPartyFileChecksumRepository;
     this.aiTranslationConfiguration = aiTranslationConfiguration;
+    this.rewriteRuleProcessor = rewriteRuleProcessor;
     this.aiTranslationService = aiTranslationService;
   }
 
@@ -218,7 +224,13 @@ void pull(
                   repositoryLocale -> {
                     String smartlingLocale = getSmartlingLocale(localeMapping, repositoryLocale);
                     SmartlingParsedFileResponse<ImmutableList<TextUnitDTO>> parsedFile =
-                        getLocalizedFileContent(projectId, file, smartlingLocale, false);
+                        getLocalizedFileContent(
+                            projectId,
+                            file,
+                            smartlingLocale,
+                            false,
+                            repository.getId(),
+                            repositoryLocale.getLocale());
 
                     if (isDeltaPull
                         && isFileEqualToPreviousRun(
@@ -248,7 +260,14 @@ && isFileEqualToPreviousRun(
                     // call to Smartling with includeOriginalStrings set to true and compare the
                     // results.
                     if (hasEmptyTranslations(textUnitDTOS)) {
-                      parsedFile = getLocalizedFileContent(projectId, file, smartlingLocale, true);
+                      parsedFile =
+                          getLocalizedFileContent(
+                              projectId,
+                              file,
+                              smartlingLocale,
+                              true,
+                              repository.getId(),
+                              repositoryLocale.getLocale());
                       ImmutableList<TextUnitDTO> textUnitDTOSWithOriginalStrings =
                           parsedFile.data();
                       textUnitDTOS =
@@ -643,12 +662,20 @@ boolean hasEmptyTranslations(ImmutableList<TextUnitDTO> textUnitDTOS) {
   }
 
   SmartlingParsedFileResponse<ImmutableList<TextUnitDTO>> getLocalizedFileContent(
-      String projectId, File file, String smartlingLocale, boolean includeOriginalStrings) {
+      String projectId,
+      File file,
+      String smartlingLocale,
+      boolean includeOriginalStrings,
+      Long repositoryId,
+      Locale locale) {
     return Mono.fromCallable(
             () -> {
               String content =
                   smartlingClient.downloadPublishedFile(
                       projectId, smartlingLocale, file.getFileUri(), includeOriginalStrings);
+              content =
+                  rewriteRuleProcessor.processExactMatches(
+                      content, repositoryId, locale.getId(), locale.getBcp47Tag());
               ImmutableList<TextUnitDTO> textUnitDTOS =
                   smartlingJsonConverter.jsonStringToTextUnitDTOs(content, TextUnitDTO::setTarget);
               return new SmartlingParsedFileResponse<>(content, textUnitDTOS);

webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullBatchFileJob.java

@@ -55,7 +55,8 @@ private void scheduleLocaleFileJob(
     String fileName =
         getOutputSourceFile(input.getBatchNumber(), repository.getName(), input.getFilePrefix());
     smartlingPullLocaleFileJobInput.setRepositoryName(input.getRepositoryName());
-    smartlingPullLocaleFileJobInput.setLocaleId(locale.getId());
+    smartlingPullLocaleFileJobInput.setRepositoryId(repository.getId());
+    smartlingPullLocaleFileJobInput.setLocaleId(locale.getLocale().getId());
     smartlingPullLocaleFileJobInput.setSmartlingFilePrefix(input.getFilePrefix());
     smartlingPullLocaleFileJobInput.setFileName(fileName);
     smartlingPullLocaleFileJobInput.setPluralSeparator(input.getPluralSeparator());

webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJob.java

@@ -9,6 +9,7 @@
 import com.box.l10n.mojito.quartz.QuartzPollableJob;
 import com.box.l10n.mojito.service.locale.LocaleRepository;
 import com.box.l10n.mojito.service.repository.RepositoryRepository;
+import com.box.l10n.mojito.service.rewriterule.RewriteRuleProcessor;
 import com.box.l10n.mojito.service.thirdparty.ThirdPartyFileChecksumRepository;
 import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingParsedFileResponse;
 import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingPluralFix;
@@ -41,6 +42,8 @@ public class SmartlingPullLocaleFileJob
 
   @Autowired SmartlingClient smartlingClient;
 
+  @Autowired RewriteRuleProcessor rewriteRuleProcessor;
+
   @Autowired LocaleMappingHelper localeMappingHelper;
 
   @Override
@@ -70,6 +73,11 @@ public Void call(SmartlingPullLocaleFileJobInput input) throws Exception {
                     String content =
                         smartlingClient.downloadPublishedFile(
                             input.getSmartlingProjectId(), smartlingLocale, fileName, false);
+
+                    content =
+                        rewriteRuleProcessor.processExactMatches(
+                            content, input.getRepositoryId(), input.getLocaleId(), localeTag);
+
                     return new SmartlingParsedFileResponse<AndroidStringDocument>(
                         content, AndroidStringDocumentReader.fromText(content));
                   })

webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJobInput.java

@@ -4,6 +4,8 @@ public class SmartlingPullLocaleFileJobInput {
 
   String repositoryName;
 
+  long repositoryId;
+
   long localeId;
 
   String localeBcp47Tag;
@@ -88,6 +90,14 @@ public void setDryRun(boolean dryRun) {
     isDryRun = dryRun;
   }
 
+  public long getRepositoryId() {
+    return repositoryId;
+  }
+
+  public void setRepositoryId(long repositoryId) {
+    this.repositoryId = repositoryId;
+  }
+
   public long getLocaleId() {
     return localeId;
   }