SOLR-4587: integrate lucene-monitor into solr

gradle/libs.versions.toml

@@ -279,6 +279,7 @@ apache-lucene-grouping = { module = "org.apache.lucene:lucene-grouping", version
 apache-lucene-highlighter = { module = "org.apache.lucene:lucene-highlighter", version.ref = "apache-lucene" }
 apache-lucene-join = { module = "org.apache.lucene:lucene-join", version.ref = "apache-lucene" }
 apache-lucene-misc = { module = "org.apache.lucene:lucene-misc", version.ref = "apache-lucene" }
+apache-lucene-monitor = { module = "org.apache.lucene:lucene-monitor", version.ref = "apache-lucene" }
 apache-lucene-queries = { module = "org.apache.lucene:lucene-queries", version.ref = "apache-lucene" }
 apache-lucene-queryparser = { module = "org.apache.lucene:lucene-queryparser", version.ref = "apache-lucene" }
 apache-lucene-sandbox = { module = "org.apache.lucene:lucene-sandbox", version.ref = "apache-lucene" }

settings.gradle

@@ -56,6 +56,7 @@ include "solr:modules:ltr"
 include "solr:modules:s3-repository"
 include "solr:modules:scripting"
 include "solr:modules:sql"
+include "solr:modules:saved-search"
 include "solr:webapp"
 include "solr:benchmark"
 include "solr:test-framework"

solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java

@@ -111,7 +111,11 @@ public class SolrResourceLoader
     "security.cert.",
     "handler.sql.",
     "crossdc.handler.",
-    "crossdc.update.processor."
+    "crossdc.update.processor.",
+    "savedsearch.",
+    "savedsearch.cache.",
+    "savedsearch.search.",
+    "savedsearch.update."
   };
   private static final Charset UTF_8 = StandardCharsets.UTF_8;
   public static final String SOLR_RESOURCELOADING_RESTRICTED_ENABLED_PARAM =

solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java

@@ -116,6 +116,14 @@ public void process(ResponseBuilder rb) throws IOException {
         info.add("facet-debug", fdebug.getFacetDebugInfo());
       }
 
+      CustomDebugInfoSources customDebugInfoSources =
+          (CustomDebugInfoSources) rb.req.getContext().get(CustomDebugInfoSources.KEY);
+      if (customDebugInfoSources != null) {
+        for (var customDebugInfoSource : customDebugInfoSources.list) {
+          info.add(customDebugInfoSource.name, customDebugInfoSource.values);
+        }
+      }
+
       if (rb.req.getJSON() != null) {
         info.add(JSON, rb.req.getJSON());
       }
@@ -140,6 +148,28 @@ public void process(ResponseBuilder rb) throws IOException {
     }
   }
 
+  public static class CustomDebugInfoSources {
+
+    public static final String KEY = "CustomDebugInfoSources";
+
+    private final List<CustomDebugInfoSource> list = new ArrayList<>();
+
+    public void add(CustomDebugInfoSource source) {
+      list.add(source);
+    }
+  }
+
+  public static class CustomDebugInfoSource {
+
+    private final String name;
+    private final SimpleOrderedMap<Object> values;
+
+    public CustomDebugInfoSource(String name, SimpleOrderedMap<Object> values) {
+      this.name = name;
+      this.values = values;
+    }
+  }
+
   private void doDebugTrack(ResponseBuilder rb) {
     final String rid = rb.req.getParams().get(CommonParams.REQUEST_ID);
     rb.addDebug(rid, "track", CommonParams.REQUEST_ID); // to see it in the response

solr/licenses/lucene-monitor-10.2.1.jar.sha1

@@ -0,0 +1 @@
+28f441a84d0b5e2ae11e1585e03144ccd7ad1f0c

solr/modules/saved-search/build.gradle

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply plugin: 'java-library'
+
+description = 'Apache Solr Monitor'
+
+dependencies {
+
+    implementation project(":solr:core")
+    implementation project(":solr:solrj")
+    implementation libs.apache.lucene.core
+    implementation libs.apache.lucene.monitor
+    implementation libs.benmanes.caffeine
+    implementation libs.dropwizard.metrics.core
+    testImplementation project(':solr:test-framework')
+    testImplementation libs.junit.junit
+}
+
+

solr/modules/saved-search/src/java/org/apache/lucene/monitor/Visitors.java

@@ -0,0 +1,85 @@
+/*
+ *
+ *  *
+ *  *  * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  *  * contributor license agreements.  See the NOTICE file distributed with
+ *  *  * this work for additional information regarding copyright ownership.
+ *  *  * The ASF licenses this file to You under the Apache License, Version 2.0
+ *  *  * (the "License"); you may not use this file except in compliance with
+ *  *  * the License.  You may obtain a copy of the License at
+ *  *  *
+ *  *  *     http://www.apache.org/licenses/LICENSE-2.0
+ *  *  *
+ *  *  * Unless required by applicable law or agreed to in writing, software
+ *  *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  *  * See the License for the specific language governing permissions and
+ *  *  * limitations under the License.
+ *  *
+ *
+ */
+
+package org.apache.lucene.monitor;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.function.BiPredicate;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.util.BytesRef;
+
+public class Visitors {
+
+  public static class DocumentBatchVisitor implements AutoCloseable, Supplier<LeafReader> {
+
+    private final DocumentBatch batch;
+    private final List<Document> docs;
+
+    private DocumentBatchVisitor(DocumentBatch batch, List<Document> docs) {
+      this.batch = batch;
+      this.docs = docs;
+    }
+
+    public static DocumentBatchVisitor of(Analyzer analyzer, List<Document> docs) {
+      return new DocumentBatchVisitor(
+          DocumentBatch.of(analyzer, docs.toArray(new Document[0])), docs);
+    }
+
+    @Override
+    public void close() throws IOException {
+      batch.close();
+    }
+
+    @Override
+    public LeafReader get() {
+      return batch.get();
+    }
+
+    public int size() {
+      return docs.size();
+    }
+
+    @Override
+    public String toString() {
+      return docs.stream().map(Document::toString).collect(Collectors.joining(" "));
+    }
+  }
+
+  public static class QueryTermFilterVisitor implements BiPredicate<String, BytesRef> {
+
+    private final QueryIndex.QueryTermFilter queryTermFilter;
+
+    public QueryTermFilterVisitor(IndexReader reader) throws IOException {
+      this.queryTermFilter = new QueryIndex.QueryTermFilter(reader);
+    }
+
+    @Override
+    public boolean test(String field, BytesRef bytesRef) {
+      return queryTermFilter.test(field, bytesRef);
+    }
+  }
+}

solr/modules/saved-search/src/java/org/apache/solr/savedsearch/AliasingPresearcher.java

@@ -0,0 +1,176 @@
+/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  * contributor license agreements.  See the NOTICE file distributed with
+ *  * this work for additional information regarding copyright ownership.
+ *  * The ASF licenses this file to You under the Apache License, Version 2.0
+ *  * (the "License"); you may not use this file except in compliance with
+ *  * the License.  You may obtain a copy of the License at
+ *  *
+ *  *     http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package org.apache.solr.savedsearch;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.BiPredicate;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.monitor.CustomQueryHandler;
+import org.apache.lucene.monitor.MultipassTermFilteredPresearcher;
+import org.apache.lucene.monitor.Presearcher;
+import org.apache.lucene.monitor.TermFilteredPresearcher;
+import org.apache.lucene.monitor.TermWeightor;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * The motivation behind this is to map each document field to its corresponding, presearcher-owned
+ * equivalent. By prefixing we ensure "presearcher fields" don't inadvertently affect the scores of
+ * "real" document fields. Another issue is that the presearcher can dynamically create field
+ * configurations for a given field name that might not be compatible with the pre-existing schema
+ * definition, see {\@link
+ * org.apache.solr.savedsearch.SingleCoreSavedSearchTest#coexistWithRegularDocumentsTest}. In the
+ * case of {@link org.apache.lucene.monitor.MultipassTermFilteredPresearcher#field(String, int)},
+ * the presearcher is capable of creating new fields by adding an ordinal suffix to an existing
+ * field name. This could also clash with user-defined name patterns, hence the existence of this
+ * class, which allows users to alias presearcher fields as they see fit.
+ */
+public class AliasingPresearcher extends Presearcher {
+
+  private final Presearcher aliasingPresearcher;
+  private final String prefix;
+
+  AliasingPresearcher(Presearcher aliasingPresearcher, String prefix) {
+    this.aliasingPresearcher = aliasingPresearcher;
+    this.prefix = prefix;
+  }
+
+  @Override
+  public Query buildQuery(LeafReader reader, BiPredicate<String, BytesRef> termAcceptor) {
+    BiPredicate<String, BytesRef> aliasingTermAcceptor =
+        (fieldName, term) -> termAcceptor.test(prefix + fieldName, term);
+    return aliasingPresearcher.buildQuery(reader, aliasingTermAcceptor);
+  }
+
+  @Override
+  public Document indexQuery(Query query, Map<String, String> metadata) {
+    var document = aliasingPresearcher.indexQuery(query, metadata);
+    return alias(document);
+  }
+
+  public String getPrefix() {
+    return prefix;
+  }
+
+  private Document alias(Document in) {
+    Document out = new Document();
+    for (var field : in) {
+      if (!TermFilteredPresearcher.ANYTOKEN_FIELD.equals(field.name())
+          && field instanceof Field
+          && ((Field) field).tokenStreamValue() != null) {
+        out.add(
+            new Field(
+                prefix + field.name(), ((Field) field).tokenStreamValue(), field.fieldType()));
+      } else {
+        out.add(field);
+      }
+    }
+    return out;
+  }
+
+  public static class MultiPassTermFiltered extends MultipassTermFilteredPresearcher {
+
+    private final String prefix;
+
+    private MultiPassTermFiltered(
+        int passes,
+        float minWeight,
+        TermWeightor weightor,
+        List<CustomQueryHandler> queryHandlers,
+        Set<String> filterFields,
+        String prefix) {
+      super(passes, minWeight, weightor, queryHandlers, filterFields);
+      this.prefix = prefix;
+    }
+
+    @Override
+    protected DocumentQueryBuilder getQueryBuilder() {
+      return new TermFiltered.AliasingDocumentQueryBuilder(super.getQueryBuilder(), prefix);
+    }
+
+    public static AliasingPresearcher build(
+        int passes,
+        float minWeight,
+        TermWeightor weightor,
+        List<CustomQueryHandler> queryHandlers,
+        Set<String> filterFields,
+        String prefix) {
+      return new AliasingPresearcher(
+          new MultiPassTermFiltered(
+              passes, minWeight, weightor, queryHandlers, filterFields, prefix),
+          prefix);
+    }
+  }
+
+  public static class TermFiltered extends TermFilteredPresearcher {
+
+    private final String prefix;
+
+    TermFiltered(
+        TermWeightor weightor,
+        List<CustomQueryHandler> customQueryHandlers,
+        Set<String> filterFields,
+        String prefix) {
+      super(weightor, customQueryHandlers, filterFields);
+      this.prefix = prefix;
+    }
+
+    public static AliasingPresearcher build(
+        TermWeightor weightor,
+        List<CustomQueryHandler> queryHandlers,
+        Set<String> filterFields,
+        String prefix) {
+      return new AliasingPresearcher(
+          new TermFiltered(weightor, queryHandlers, filterFields, prefix), prefix);
+    }
+
+    @Override
+    protected DocumentQueryBuilder getQueryBuilder() {
+      return new AliasingDocumentQueryBuilder(super.getQueryBuilder(), prefix);
+    }
+
+    static class AliasingDocumentQueryBuilder implements DocumentQueryBuilder {
+
+      private final DocumentQueryBuilder documentQueryBuilder;
+      private final String prefix;
+
+      public AliasingDocumentQueryBuilder(
+          DocumentQueryBuilder documentQueryBuilder, String prefix) {
+        this.documentQueryBuilder = documentQueryBuilder;
+        this.prefix = prefix;
+      }
+
+      @Override
+      public void addTerm(String field, BytesRef term) throws IOException {
+        documentQueryBuilder.addTerm(prefix + field, term);
+      }
+
+      @Override
+      public Query build() {
+        return documentQueryBuilder.build();
+      }
+    }
+  }
+}