CD-7540 - ES migration from 8 to 9

build.gradle

@@ -221,7 +221,7 @@ subprojects {
   ext {
     protobufVersion = '4.28.2'
     springVersion = '5.3.39'
-    elasticSearchClientVersion = '7.17.24'
+    elasticSearchJavaClientVersion = '9.2.8' // New Java API Client
   }
 
   sonar {
@@ -432,15 +432,13 @@ subprojects {
         entry 'jetty-server'
         entry 'jetty-servlet'
       }
-      dependency("org.elasticsearch.client:elasticsearch-rest-high-level-client:${elasticSearchClientVersion}") {
-        exclude 'org.apache.logging.log4j:log4j-core'
-        exclude 'org.elasticsearch:jna'
-        exclude 'com.fasterxml.jackson.core:jackson-core'
-        exclude 'org.lz4:lz4-java'
-      }
       dependency 'com.fasterxml.jackson.core:jackson-core:2.21.2'
-      dependency "org.elasticsearch.plugin:transport-netty4-client:${elasticSearchClientVersion}"
       dependency 'org.elasticsearch:mocksocket:1.2'
+
+      // New Elasticsearch Java API Client (8.x) - For migration
+      dependency "co.elastic.clients:elasticsearch-java:${elasticSearchJavaClientVersion}"
+      dependency "org.elasticsearch.client:elasticsearch-rest-client:${elasticSearchJavaClientVersion}"
+      dependency 'org.eclipse.jgit:org.eclipse.jgit:7.4.0.202509020913-r'
       dependency 'org.eclipse.jgit:org.eclipse.jgit:7.3.0.202506031305-r'
       dependency "org.codelibs.elasticsearch.module:analysis-common:7.17.22"
       dependency "org.codelibs.elasticsearch.module:reindex:7.17.22"

codescan-application/build.gradle

@@ -47,7 +47,6 @@ dependencies {
     // please keep list ordered
     api 'org.slf4j:slf4j-api'
 
-    api 'org.elasticsearch.client:elasticsearch-rest-high-level-client'
     api 'com.fasterxml.jackson.core:jackson-core'
     api 'org.sonarsource.api.plugin:sonar-plugin-api'
     api project(':server:sonar-ce')
@@ -128,7 +127,7 @@ zip {
     //When the archive size increases due to dependencies, the expected size should be updated as well.
     //Bump the expected size by at least 10 more megabytes than what is strictly needed, this in conjunction with the
     //tolerance will allow for some growth in the archive size.
-    def expectedSize = 300_000_000
+    def expectedSize = 350_000_000
     //We set a tolerance of 15MB to avoid failing the build for small differences in the archive size.
     def tolerance = 30_000_000
     def minArchiveSize = expectedSize - tolerance

gradle.properties

@@ -11,7 +11,7 @@ projectTitle=SonarQube
 org.gradle.jvmargs=-Xmx2048m
 org.gradle.caching=true
 org.gradle.vfs.watch=true
-elasticSearchServerVersion=8.14.3
+elasticSearchServerVersion=9.2.8
 projectType=application
 jre_release_name=jdk-17.0.11+9
 deployCommunity=true

server/sonar-main/build.gradle

@@ -17,7 +17,7 @@ dependencies {
   api 'org.apache.commons:commons-lang3'
   api 'org.apache.logging.log4j:log4j-to-slf4j'
   api 'org.apache.logging.log4j:log4j-api'
-  api 'org.elasticsearch.client:elasticsearch-rest-high-level-client'
+  api 'co.elastic.clients:elasticsearch-java'
   api 'com.fasterxml.jackson.core:jackson-core'
   api 'org.slf4j:slf4j-api'
   api 'org.yaml:snakeyaml'

server/sonar-server-common/build.gradle

@@ -18,7 +18,8 @@ dependencies {
   api 'org.apache.commons:commons-email'
   api 'org.apache.commons:commons-lang3'
   api 'org.bouncycastle:bc-fips'
-  api 'org.elasticsearch.client:elasticsearch-rest-high-level-client'
+  api 'co.elastic.clients:elasticsearch-java'
+  api 'org.elasticsearch.client:elasticsearch-rest-client'
   api 'com.fasterxml.jackson.core:jackson-core'
   api 'org.slf4j:slf4j-api'
   api 'org.sonarsource.api.plugin:sonar-plugin-api'
@@ -31,7 +32,6 @@ dependencies {
   api project(':sonar-ws')
 
   compileOnlyApi 'com.github.spotbugs:spotbugs-annotations'
-  testImplementation 'org.elasticsearch.plugin:transport-netty4-client'
   testImplementation 'ch.qos.logback:logback-core'
   testImplementation 'com.github.spotbugs:spotbugs-annotations'
   testImplementation 'org.subethamail:subethasmtp'
@@ -59,7 +59,6 @@ dependencies {
   testFixturesApi testFixtures(project(':server:sonar-db-dao'))
 
   testFixturesCompileOnly 'com.github.spotbugs:spotbugs-annotations'
-  testFixturesImplementation 'org.elasticsearch.plugin:transport-netty4-client'
   testFixturesImplementation 'org.codelibs.elasticsearch.module:analysis-common'
   testFixturesImplementation 'org.codelibs.elasticsearch.module:reindex'
   testFixturesImplementation 'org.elasticsearch:mocksocket'

server/sonar-server-common/src/it/java/org/sonar/server/es/BulkIndexerIT.java

@@ -19,18 +19,12 @@
  */
 package org.sonar.server.es;
 
-import java.io.IOException;
+import co.elastic.clients.elasticsearch.core.IndexRequest;
+import co.elastic.clients.elasticsearch.core.SearchRequest;
+import co.elastic.clients.elasticsearch.indices.GetIndicesSettingsResponse;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
-import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.junit.Rule;
 import org.junit.Test;
 import org.slf4j.event.Level;
@@ -58,18 +52,20 @@ public class BulkIndexerIT {
   @Rule
   public LogTester logTester = new LogTester();
 
+  @SuppressWarnings({"rawtypes", "unchecked"})
   @Test
   public void index_nothing() {
-    BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR);
+    BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR, Object.class);
     indexer.start();
     indexer.stop();
 
     assertThat(count()).isZero();
   }
 
+  @SuppressWarnings({"rawtypes", "unchecked"})
   @Test
   public void index_documents() {
-    BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR);
+    BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR, Object.class);
     indexer.start();
     indexer.add(newIndexRequest(42));
     indexer.add(newIndexRequest(78));
@@ -82,12 +78,13 @@ public void index_documents() {
     assertThat(count()).isEqualTo(2);
   }
 
+  @SuppressWarnings({"rawtypes", "unchecked"})
   @Test
   public void large_indexing() {
     // index has one replica
     assertThat(replicas()).isOne();
 
-    BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.LARGE);
+    BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.LARGE, Object.class);
     indexer.start();
 
     // replicas are temporarily disabled
@@ -118,17 +115,27 @@ public void bulk_delete() {
     es.putDocuments(TYPE_FAKE, docs);
     assertThat(count()).isEqualTo(max);
 
-    SearchRequest req = EsClient.prepareSearch(TYPE_FAKE)
-      .source(new SearchSourceBuilder().query(QueryBuilders.rangeQuery(FakeIndexDefinition.INT_FIELD).gte(removeFrom)));
-    BulkIndexer.delete(es.client(), TYPE_FAKE, req);
+    SearchRequest req = SearchRequest.of(b -> b
+      .index(TYPE_FAKE.getIndex().getName())
+      .query(q -> q
+        .range(r -> r
+          .number(nb -> nb
+            .field(FakeIndexDefinition.INT_FIELD)
+            .gte((double)removeFrom)
+          )
+        )
+      )
+    );
+    BulkIndexer.delete(es.client(), TYPE_FAKE, req, Object.class);
 
     assertThat(count()).isEqualTo(removeFrom);
   }
 
+  @SuppressWarnings({"rawtypes", "unchecked"})
   @Test
   public void listener_is_called_on_successful_requests() {
     FakeListener listener = new FakeListener();
-    BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR, listener);
+    BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR, listener, Object.class);
     indexer.start();
     indexer.addDeletion(TYPE_FAKE, "foo");
     indexer.stop();
@@ -138,10 +145,11 @@ public void listener_is_called_on_successful_requests() {
     assertThat(listener.calledResult.getTotal()).isOne();
   }
 
+  @SuppressWarnings({"rawtypes", "unchecked"})
   @Test
   public void listener_is_called_even_if_deleting_a_doc_that_does_not_exist() {
     FakeListener listener = new FakeListener();
-    BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR, listener);
+    BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR, listener, Object.class);
     indexer.start();
     indexer.add(newIndexRequestWithDocId("foo"));
     indexer.add(newIndexRequestWithDocId("bar"));
@@ -152,24 +160,31 @@ public void listener_is_called_even_if_deleting_a_doc_that_does_not_exist() {
     assertThat(listener.calledResult.getTotal()).isEqualTo(2);
   }
 
+  @SuppressWarnings({"rawtypes", "unchecked"})
   @Test
   public void listener_is_not_called_with_errors() {
     FakeListener listener = new FakeListener();
-    BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR, listener);
+    BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR, listener, Object.class);
     indexer.start();
     indexer.add(newIndexRequestWithDocId("foo"));
-    indexer.add(new IndexRequest("index_does_not_exist").id("bar").source(emptyMap()));
+    indexer.add(IndexRequest.of(b -> b
+        .index("index_does_not_exist")
+        .id("bar")
+        .document(emptyMap())
+      )
+    );
     indexer.stop();
     assertThat(listener.calledDocIds).containsExactly(newDocId(EXCPECTED_TYPE_FAKE, "foo"));
     assertThat(listener.calledResult.getSuccess()).isOne();
     assertThat(listener.calledResult.getTotal()).isEqualTo(2);
   }
 
+  @SuppressWarnings({"rawtypes", "unchecked"})
   @Test
   public void log_requests_when_TRACE_level_is_enabled() {
     logTester.setLevel(LoggerLevel.TRACE);
 
-    BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR, new FakeListener());
+    BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR, new FakeListener(), Object.class);
     indexer.start();
     indexer.add(newIndexRequestWithDocId("foo"));
     indexer.addDeletion(TYPE_FAKE, "foo");
@@ -203,24 +218,25 @@ private long count() {
   }
 
   private int replicas() {
-    try {
-      GetSettingsResponse settingsResp = es.client().nativeClient().indices()
-          .getSettings(new GetSettingsRequest().indices(INDEX), RequestOptions.DEFAULT);
-      return Integer.parseInt(settingsResp.getSetting(INDEX, IndexMetadata.SETTING_NUMBER_OF_REPLICAS));
-    } catch (IOException e) {
-      throw new IllegalStateException("Could not get index settings", e);
-    }
+    GetIndicesSettingsResponse settingsResp =
+      es.client().getSettingsV2(req -> req.index(INDEX));
+    return Integer.parseInt(settingsResp.get(INDEX).settings().index().numberOfReplicas());
   }
-
+  @SuppressWarnings("rawtypes")
   private IndexRequest newIndexRequest(int intField) {
-    return new IndexRequest(INDEX)
-      .source(Map.of(FakeIndexDefinition.INT_FIELD, intField));
+    return IndexRequest.of(b -> b
+      .index(INDEX)
+      .document(Map.of(FakeIndexDefinition.INT_FIELD, intField))
+    );
   }
 
+  @SuppressWarnings("rawtypes")
   private IndexRequest newIndexRequestWithDocId(String id) {
-    return new IndexRequest(INDEX)
+    return IndexRequest.of(b -> b
+      .index(INDEX)
       .id(id)
-      .source(Map.of(FakeIndexDefinition.INT_FIELD, 42));
+      .document(Map.of(FakeIndexDefinition.INT_FIELD, 42))
+    );
   }
 
   private static DocId newDocId(IndexType.IndexMainType mainType, String id) {

server/sonar-server-common/src/it/java/org/sonar/server/view/index/ViewIndexIT.java

@@ -20,24 +20,24 @@
 package org.sonar.server.view.index;
 
 import java.util.List;
-import org.junit.Rule;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
 import org.sonar.server.es.EsTester;
 
 import static com.google.common.collect.Lists.newArrayList;
 import static java.util.Collections.singletonList;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.sonar.server.view.index.ViewIndexDefinition.TYPE_VIEW;
 
-public class ViewIndexIT {
+class ViewIndexIT {
 
-  @Rule
+  @RegisterExtension
   public EsTester es = EsTester.create();
 
-  private ViewIndex index = new ViewIndex(es.client());
+  private final ViewIndex index = new ViewIndex(es.client());
 
   @Test
-  public void find_all_view_uuids() {
+  void find_all_view_uuids() {
     ViewDoc view1 = new ViewDoc().setUuid("UUID1").setProjectBranchUuids(singletonList("P1"));
     ViewDoc view2 = new ViewDoc().setUuid("UUID2").setProjectBranchUuids(singletonList("P2"));
     es.putDocuments(TYPE_VIEW, view1);
@@ -49,7 +49,7 @@ public void find_all_view_uuids() {
   }
 
   @Test
-  public void not_find_all_view_uuids() {
+  void not_find_all_view_uuids() {
     List<String> result = newArrayList(index.findAllViewUuids());
 
     assertThat(result).isEmpty();

server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentHit.java

@@ -19,49 +19,31 @@
  */
 package org.sonar.server.component.index;
 
+import co.elastic.clients.elasticsearch.core.search.Hit;
 import java.util.List;
 import java.util.Optional;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.search.SearchHit;
+import javax.annotation.Nullable;
 
-import static java.util.Arrays.stream;
 import static java.util.Optional.ofNullable;
 import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_NAME;
 
-public class ComponentHit {
-
-  private final String uuid;
-  private final Optional<String> highlightedText;
+public record ComponentHit(@Nullable String uuid, @Nullable String highlightedText) {
 
   public ComponentHit(String uuid) {
-    this.uuid = uuid;
-    this.highlightedText = Optional.empty();
-  }
-
-  public ComponentHit(SearchHit hit) {
-    this.uuid = hit.getId();
-    this.highlightedText = getHighlightedText(hit);
-  }
-
-  private static Optional<String> getHighlightedText(SearchHit hit) {
-    return ofNullable(hit.getHighlightFields())
-      .flatMap(fields -> ofNullable(fields.get(FIELD_NAME)))
-      .flatMap(field -> ofNullable(field.getFragments()))
-      .flatMap(fragments -> stream(fragments).findFirst())
-      .map(Text::string);
-  }
-
-  public String getUuid() {
-    return uuid;
+    this(uuid, null);
   }
 
-  public static List<ComponentHit> fromSearchHits(SearchHit... hits) {
-    return stream(hits)
-      .map(ComponentHit::new)
+  public static <T> List<ComponentHit> fromSearchHitsV2(List<Hit<T>> hits) {
+    return hits.stream()
+      .map(ComponentHit::fromHitV2)
       .toList();
   }
 
-  public Optional<String> getHighlightedText() {
-    return highlightedText;
+  private static <T> ComponentHit fromHitV2(Hit<T> hit) {
+    Optional<String> highlightedText = ofNullable(hit.highlight())
+      .flatMap(highlightMap -> ofNullable(highlightMap.get(FIELD_NAME)))
+      .flatMap(fragments -> fragments.stream().findFirst());
+
+    return new ComponentHit(hit.id(), highlightedText.orElse(null));
   }
 }