diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b795bb4b62fd..0cbf0836fb88 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -279,6 +279,7 @@ apache-lucene-grouping = { module = "org.apache.lucene:lucene-grouping", version apache-lucene-highlighter = { module = "org.apache.lucene:lucene-highlighter", version.ref = "apache-lucene" } apache-lucene-join = { module = "org.apache.lucene:lucene-join", version.ref = "apache-lucene" } apache-lucene-misc = { module = "org.apache.lucene:lucene-misc", version.ref = "apache-lucene" } +apache-lucene-monitor = { module = "org.apache.lucene:lucene-monitor", version.ref = "apache-lucene" } apache-lucene-queries = { module = "org.apache.lucene:lucene-queries", version.ref = "apache-lucene" } apache-lucene-queryparser = { module = "org.apache.lucene:lucene-queryparser", version.ref = "apache-lucene" } apache-lucene-sandbox = { module = "org.apache.lucene:lucene-sandbox", version.ref = "apache-lucene" } diff --git a/settings.gradle b/settings.gradle index 4a16234cc523..b05216f2c77d 100644 --- a/settings.gradle +++ b/settings.gradle @@ -56,6 +56,7 @@ include "solr:modules:ltr" include "solr:modules:s3-repository" include "solr:modules:scripting" include "solr:modules:sql" +include "solr:modules:saved-search" include "solr:webapp" include "solr:benchmark" include "solr:test-framework" diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java index 3f3d2eda3ab2..a4864e867cf7 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java +++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java @@ -111,7 +111,11 @@ public class SolrResourceLoader "security.cert.", "handler.sql.", "crossdc.handler.", - "crossdc.update.processor." + "crossdc.update.processor.", + "savedsearch.", + "savedsearch.cache.", + "savedsearch.search.", + "savedsearch.update." }; private static final Charset UTF_8 = StandardCharsets.UTF_8; public static final String SOLR_RESOURCELOADING_RESTRICTED_ENABLED_PARAM = diff --git a/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java b/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java index 7e67d1c3a752..443898ac6810 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java @@ -116,6 +116,14 @@ public void process(ResponseBuilder rb) throws IOException { info.add("facet-debug", fdebug.getFacetDebugInfo()); } + CustomDebugInfoSources customDebugInfoSources = + (CustomDebugInfoSources) rb.req.getContext().get(CustomDebugInfoSources.KEY); + if (customDebugInfoSources != null) { + for (var customDebugInfoSource : customDebugInfoSources.list) { + info.add(customDebugInfoSource.name, customDebugInfoSource.values); + } + } + if (rb.req.getJSON() != null) { info.add(JSON, rb.req.getJSON()); } @@ -140,6 +148,28 @@ public void process(ResponseBuilder rb) throws IOException { } } + public static class CustomDebugInfoSources { + + public static final String KEY = "CustomDebugInfoSources"; + + private final List list = new ArrayList<>(); + + public void add(CustomDebugInfoSource source) { + list.add(source); + } + } + + public static class CustomDebugInfoSource { + + private final String name; + private final SimpleOrderedMap values; + + public CustomDebugInfoSource(String name, SimpleOrderedMap values) { + this.name = name; + this.values = values; + } + } + private void doDebugTrack(ResponseBuilder rb) { final String rid = rb.req.getParams().get(CommonParams.REQUEST_ID); rb.addDebug(rid, "track", CommonParams.REQUEST_ID); // to see it in the response diff --git a/solr/licenses/lucene-monitor-10.2.1.jar.sha1 b/solr/licenses/lucene-monitor-10.2.1.jar.sha1 new file mode 100644 index 000000000000..6a3002ba6340 --- /dev/null +++ b/solr/licenses/lucene-monitor-10.2.1.jar.sha1 @@ -0,0 +1 @@ +28f441a84d0b5e2ae11e1585e03144ccd7ad1f0c diff --git a/solr/modules/saved-search/build.gradle b/solr/modules/saved-search/build.gradle new file mode 100644 index 000000000000..32781165e301 --- /dev/null +++ b/solr/modules/saved-search/build.gradle @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: 'java-library' + +description = 'Apache Solr Monitor' + +dependencies { + + implementation project(":solr:core") + implementation project(":solr:solrj") + implementation libs.apache.lucene.core + implementation libs.apache.lucene.monitor + implementation libs.benmanes.caffeine + implementation libs.dropwizard.metrics.core + testImplementation project(':solr:test-framework') + testImplementation libs.junit.junit +} + + diff --git a/solr/modules/saved-search/src/java/org/apache/lucene/monitor/Visitors.java b/solr/modules/saved-search/src/java/org/apache/lucene/monitor/Visitors.java new file mode 100644 index 000000000000..9242c54e6178 --- /dev/null +++ b/solr/modules/saved-search/src/java/org/apache/lucene/monitor/Visitors.java @@ -0,0 +1,85 @@ +/* + * + * * + * * * Licensed to the Apache Software Foundation (ASF) under one or more + * * * contributor license agreements. See the NOTICE file distributed with + * * * this work for additional information regarding copyright ownership. + * * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * * (the "License"); you may not use this file except in compliance with + * * * the License. You may obtain a copy of the License at + * * * + * * * http://www.apache.org/licenses/LICENSE-2.0 + * * * + * * * Unless required by applicable law or agreed to in writing, software + * * * distributed under the License is distributed on an "AS IS" BASIS, + * * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * * See the License for the specific language governing permissions and + * * * limitations under the License. + * * + * + */ + +package org.apache.lucene.monitor; + +import java.io.IOException; +import java.util.List; +import java.util.function.BiPredicate; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.util.BytesRef; + +public class Visitors { + + public static class DocumentBatchVisitor implements AutoCloseable, Supplier { + + private final DocumentBatch batch; + private final List docs; + + private DocumentBatchVisitor(DocumentBatch batch, List docs) { + this.batch = batch; + this.docs = docs; + } + + public static DocumentBatchVisitor of(Analyzer analyzer, List docs) { + return new DocumentBatchVisitor( + DocumentBatch.of(analyzer, docs.toArray(new Document[0])), docs); + } + + @Override + public void close() throws IOException { + batch.close(); + } + + @Override + public LeafReader get() { + return batch.get(); + } + + public int size() { + return docs.size(); + } + + @Override + public String toString() { + return docs.stream().map(Document::toString).collect(Collectors.joining(" ")); + } + } + + public static class QueryTermFilterVisitor implements BiPredicate { + + private final QueryIndex.QueryTermFilter queryTermFilter; + + public QueryTermFilterVisitor(IndexReader reader) throws IOException { + this.queryTermFilter = new QueryIndex.QueryTermFilter(reader); + } + + @Override + public boolean test(String field, BytesRef bytesRef) { + return queryTermFilter.test(field, bytesRef); + } + } +} diff --git a/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/AliasingPresearcher.java b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/AliasingPresearcher.java new file mode 100644 index 000000000000..ef001a1d7ed0 --- /dev/null +++ b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/AliasingPresearcher.java @@ -0,0 +1,176 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.solr.savedsearch; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.BiPredicate; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.monitor.CustomQueryHandler; +import org.apache.lucene.monitor.MultipassTermFilteredPresearcher; +import org.apache.lucene.monitor.Presearcher; +import org.apache.lucene.monitor.TermFilteredPresearcher; +import org.apache.lucene.monitor.TermWeightor; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; + +/** + * The motivation behind this is to map each document field to its corresponding, presearcher-owned + * equivalent. By prefixing we ensure "presearcher fields" don't inadvertently affect the scores of + * "real" document fields. Another issue is that the presearcher can dynamically create field + * configurations for a given field name that might not be compatible with the pre-existing schema + * definition, see {\@link + * org.apache.solr.savedsearch.SingleCoreSavedSearchTest#coexistWithRegularDocumentsTest}. In the + * case of {@link org.apache.lucene.monitor.MultipassTermFilteredPresearcher#field(String, int)}, + * the presearcher is capable of creating new fields by adding an ordinal suffix to an existing + * field name. This could also clash with user-defined name patterns, hence the existence of this + * class, which allows users to alias presearcher fields as they see fit. + */ +public class AliasingPresearcher extends Presearcher { + + private final Presearcher aliasingPresearcher; + private final String prefix; + + AliasingPresearcher(Presearcher aliasingPresearcher, String prefix) { + this.aliasingPresearcher = aliasingPresearcher; + this.prefix = prefix; + } + + @Override + public Query buildQuery(LeafReader reader, BiPredicate termAcceptor) { + BiPredicate aliasingTermAcceptor = + (fieldName, term) -> termAcceptor.test(prefix + fieldName, term); + return aliasingPresearcher.buildQuery(reader, aliasingTermAcceptor); + } + + @Override + public Document indexQuery(Query query, Map metadata) { + var document = aliasingPresearcher.indexQuery(query, metadata); + return alias(document); + } + + public String getPrefix() { + return prefix; + } + + private Document alias(Document in) { + Document out = new Document(); + for (var field : in) { + if (!TermFilteredPresearcher.ANYTOKEN_FIELD.equals(field.name()) + && field instanceof Field + && ((Field) field).tokenStreamValue() != null) { + out.add( + new Field( + prefix + field.name(), ((Field) field).tokenStreamValue(), field.fieldType())); + } else { + out.add(field); + } + } + return out; + } + + public static class MultiPassTermFiltered extends MultipassTermFilteredPresearcher { + + private final String prefix; + + private MultiPassTermFiltered( + int passes, + float minWeight, + TermWeightor weightor, + List queryHandlers, + Set filterFields, + String prefix) { + super(passes, minWeight, weightor, queryHandlers, filterFields); + this.prefix = prefix; + } + + @Override + protected DocumentQueryBuilder getQueryBuilder() { + return new TermFiltered.AliasingDocumentQueryBuilder(super.getQueryBuilder(), prefix); + } + + public static AliasingPresearcher build( + int passes, + float minWeight, + TermWeightor weightor, + List queryHandlers, + Set filterFields, + String prefix) { + return new AliasingPresearcher( + new MultiPassTermFiltered( + passes, minWeight, weightor, queryHandlers, filterFields, prefix), + prefix); + } + } + + public static class TermFiltered extends TermFilteredPresearcher { + + private final String prefix; + + TermFiltered( + TermWeightor weightor, + List customQueryHandlers, + Set filterFields, + String prefix) { + super(weightor, customQueryHandlers, filterFields); + this.prefix = prefix; + } + + public static AliasingPresearcher build( + TermWeightor weightor, + List queryHandlers, + Set filterFields, + String prefix) { + return new AliasingPresearcher( + new TermFiltered(weightor, queryHandlers, filterFields, prefix), prefix); + } + + @Override + protected DocumentQueryBuilder getQueryBuilder() { + return new AliasingDocumentQueryBuilder(super.getQueryBuilder(), prefix); + } + + static class AliasingDocumentQueryBuilder implements DocumentQueryBuilder { + + private final DocumentQueryBuilder documentQueryBuilder; + private final String prefix; + + public AliasingDocumentQueryBuilder( + DocumentQueryBuilder documentQueryBuilder, String prefix) { + this.documentQueryBuilder = documentQueryBuilder; + this.prefix = prefix; + } + + @Override + public void addTerm(String field, BytesRef term) throws IOException { + documentQueryBuilder.addTerm(prefix + field, term); + } + + @Override + public Query build() { + return documentQueryBuilder.build(); + } + } + } +} diff --git a/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/SavedSearchDataValues.java b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/SavedSearchDataValues.java new file mode 100644 index 000000000000..186a094e7de7 --- /dev/null +++ b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/SavedSearchDataValues.java @@ -0,0 +1,131 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.solr.savedsearch; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.monitor.MonitorQuery; +import org.apache.lucene.monitor.QueryDecomposer; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Query; +import org.apache.solr.common.params.CommonParams; + +public class SavedSearchDataValues { + + public static final String QUERY_ID = "_query_id_"; + public static final String MONITOR_QUERY = "_mq_"; + + public static final Set REQUIRED_MONITOR_SCHEMA_FIELDS = Set.of(QUERY_ID, MONITOR_QUERY); + + private final SortedDocValues queryIdIt; + private final SortedDocValues cacheIdIt; + private final SortedDocValues mqIt; + private final NumericDocValues versionIt; + private final LeafReader reader; + private int currentDoc; + + public SavedSearchDataValues(LeafReaderContext context, String idFieldName) throws IOException { + reader = context.reader(); + cacheIdIt = reader.getSortedDocValues(idFieldName); + queryIdIt = reader.getSortedDocValues(QUERY_ID); + mqIt = reader.getSortedDocValues(MONITOR_QUERY); + versionIt = reader.getNumericDocValues(CommonParams.VERSION_FIELD); + currentDoc = DocIdSetIterator.NO_MORE_DOCS; + } + + public boolean advanceTo(int doc) throws IOException { + currentDoc = doc; + return cacheIdIt.advanceExact(currentDoc); + } + + public String getQueryId() throws IOException { + queryIdIt.advanceExact(currentDoc); + return queryIdIt.lookupOrd(queryIdIt.ordValue()).utf8ToString(); + } + + public String getCacheId() throws IOException { + return cacheIdIt.lookupOrd(cacheIdIt.ordValue()).utf8ToString(); + } + + public String getMq() throws IOException { + if (mqIt != null && mqIt.advanceExact(currentDoc)) { + return mqIt.lookupOrd(mqIt.ordValue()).utf8ToString(); + } + return reader.storedFields().document(currentDoc).get(MONITOR_QUERY); + } + + public long getVersion() throws IOException { + if (versionIt != null && versionIt.advanceExact(currentDoc)) { + return versionIt.longValue(); + } + return 0; + } + + public static class QueryDisjunct { + + private final String disjunctId; + private final String queryId; + private final Query matchQuery; + private final Map metadata; + + private QueryDisjunct( + String disjunctId, String queryId, Query matchQuery, Map metadata) { + this.disjunctId = disjunctId; + this.queryId = queryId; + this.matchQuery = matchQuery; + this.metadata = metadata; + } + + public static List decompose(MonitorQuery mq, QueryDecomposer decomposer) { + int upto = 0; + List disjuncts = new ArrayList<>(); + for (var disjunct : decomposer.decompose(mq.getQuery())) { + String suffix = upto == 0 ? "" : "_" + upto; + disjuncts.add( + new QueryDisjunct(mq.getId() + suffix, mq.getId(), disjunct, mq.getMetadata())); + upto++; + } + return disjuncts; + } + + public Query getMatchQuery() { + return matchQuery; + } + + public String getId() { + return disjunctId; + } + + public String getQueryId() { + return queryId; + } + + public Map getMetadata() { + return metadata; + } + } +} diff --git a/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/SavedSearchDecoder.java b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/SavedSearchDecoder.java new file mode 100644 index 000000000000..7ddf688d8fa2 --- /dev/null +++ b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/SavedSearchDecoder.java @@ -0,0 +1,59 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.solr.savedsearch; + +import java.io.IOException; +import java.util.Map; +import org.apache.lucene.monitor.MonitorQuery; +import org.apache.lucene.monitor.QueryDecomposer; +import org.apache.solr.core.SolrCore; +import org.apache.solr.savedsearch.SavedSearchDataValues.QueryDisjunct; +import org.apache.solr.savedsearch.search.ReverseSearchComponent; + +public class SavedSearchDecoder { + + private final SolrCore core; + private final QueryDecomposer queryDecomposer; + + public SavedSearchDecoder(SolrCore core) { + this.core = core; + ReverseSearchComponent rsc = + (ReverseSearchComponent) + core.getSearchComponents().get(ReverseSearchComponent.COMPONENT_NAME); + this.queryDecomposer = rsc.getQueryDecomposer(); + } + + private MonitorQuery decode(SavedSearchDataValues savedSearchDataValues) throws IOException { + String id = savedSearchDataValues.getQueryId(); + String queryStr = savedSearchDataValues.getMq(); + var query = SimpleQueryParser.parse(queryStr, core); + return new MonitorQuery(id, query, queryStr, Map.of()); + } + + public QueryDisjunct getDisjunct(SavedSearchDataValues dataValues, String cacheId) + throws IOException { + for (QueryDisjunct qce : QueryDisjunct.decompose(decode(dataValues), queryDecomposer)) { + if (qce.getId().equals(cacheId)) { + return qce; + } + } + throw new IllegalArgumentException("Corrupt monitorQuery value in index"); + } +} diff --git a/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/SimpleQueryParser.java b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/SimpleQueryParser.java new file mode 100644 index 000000000000..2d7d41626b52 --- /dev/null +++ b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/SimpleQueryParser.java @@ -0,0 +1,147 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.solr.savedsearch; + +import java.security.Principal; +import java.util.HashMap; +import java.util.Map; +import org.apache.lucene.search.Query; +import org.apache.solr.common.params.MapSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.ContentStream; +import org.apache.solr.core.SolrCore; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.search.QParser; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.SyntaxError; +import org.apache.solr.util.RTimerTree; + +/** + * Fake Solr request used only in some places which need Solr Requests, but not all methods are + * needed. + */ +public class SimpleQueryParser implements SolrQueryRequest { + + private static final SolrParams EMPTY = new MapSolrParams(Map.of()); + + private final SolrCore solrCore; + private final Map context; + private final IndexSchema schema; + private final RTimerTree timerTree; + + private SimpleQueryParser(SolrCore solrCore) { + this.solrCore = solrCore; + this.schema = solrCore.getLatestSchema(); + context = new HashMap<>(); + this.timerTree = new RTimerTree(); + } + + public static Query parse(String queryStr, SolrCore core) { + try { + return QParser.getParser(queryStr, new SimpleQueryParser(core)).parse(); + } catch (SyntaxError e) { + throw new IllegalStateException("could not parse query", e); + } + } + + @Override + public SolrParams getParams() { + return EMPTY; + } + + @Override + public void setParams(SolrParams params) { + throw new UnsupportedOperationException("SolrQueryRequest:: setParams Not supported."); + } + + @Override + public Iterable getContentStreams() { + throw new UnsupportedOperationException("SolrQueryRequest::getContentStreams Not supported."); + } + + @Override + public SolrParams getOriginalParams() { + return EMPTY; + } + + @Override + public Map getContext() { + return context; + } + + @Override + public void close() { + /* no-op */ + } + + protected final long startTime = System.nanoTime(); + + @Override + public long getStartTime() { + return startTime; + } + + @Override + public RTimerTree getRequestTimer() { + return timerTree; + } + + @Override + public SolrIndexSearcher getSearcher() { + return solrCore.getSearcher().get(); + } + + @Override + public SolrCore getCore() { + return solrCore; + } + + @Override + public IndexSchema getSchema() { + return schema; + } + + @Override + public void updateSchemaToLatest() { + throw new UnsupportedOperationException( + "SolrQueryRequest::updateSchemaToLatest Not supported."); + } + + @Override + public String getParamString() { + throw new UnsupportedOperationException("SolrQueryRequest::getParamString Not supported."); + } + + @Override + public Map getJSON() { + throw new UnsupportedOperationException("SolrQueryRequest::getJSON Not supported."); + } + + @Override + public void setJSON(Map json) { + throw new UnsupportedOperationException("SolrQueryRequest::setJSON Not supported."); + } + + @Override + public Principal getUserPrincipal() { + throw new UnsupportedOperationException("SolrQueryRequest::getUserPrincipal Not supported."); + } +} diff --git a/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/cache/DefaultSavedSearchCache.java b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/cache/DefaultSavedSearchCache.java new file mode 100644 index 000000000000..d049fc4dee5f --- /dev/null +++ b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/cache/DefaultSavedSearchCache.java @@ -0,0 +1,424 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.solr.savedsearch.cache; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.RemovalCause; +import com.github.benmanes.caffeine.cache.RemovalListener; +import java.io.IOException; +import java.time.Duration; +import java.util.Arrays; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiPredicate; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReader.CacheKey; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.monitor.Visitors.QueryTermFilterVisitor; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.core.SolrCore; +import org.apache.solr.metrics.MetricsMap; +import org.apache.solr.metrics.SolrMetricsContext; +import org.apache.solr.savedsearch.SavedSearchDataValues; +import org.apache.solr.savedsearch.SavedSearchDataValues.QueryDisjunct; +import org.apache.solr.savedsearch.SavedSearchDecoder; +import org.apache.solr.savedsearch.cache.SavedSearchCache.VersionedQueryCacheEntry; +import org.apache.solr.search.CacheRegenerator; +import org.apache.solr.search.SolrCache; +import org.apache.solr.search.SolrCacheBase; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.util.IOFunction; + +public class DefaultSavedSearchCache extends SolrCacheBase + implements SolrCache, + SavedSearchCache, + RemovalListener { + + private final AtomicReference currentStats = + new AtomicReference<>(CurrentStats.init()); + + private Cache mqCache; + private volatile BiPredicate termFilter = (__, ___) -> true; + + private SolrMetricsContext solrMetricsContext; + private long generationTimeMs; + private long priorLookups; + private long priorHits; + private long cumulativeGenerationTimeMs; + private long cumulativeDocVisits; + + private long docVisits; + // only needs to be an approximation + private Cache maxVersionMap = + Caffeine.newBuilder().weakKeys().build(); + private int initialSize; + private int maxSize; + private int maxRamMB; + private int initialRamMB; + + @Override + public VersionedQueryCacheEntry computeIfStale( + SavedSearchDataValues dataValues, SavedSearchDecoder decoder) throws IOException { + return mqCacheMap() + .compute( + dataValues.getCacheId(), + (cacheId, prevEntry) -> compute(cacheId, prevEntry, dataValues, decoder)); + } + + @Override + public BiPredicate termFilter() { + return termFilter; + } + + private Map mqCacheMap() { + return mqCache.asMap(); + } + + @Override + public Object init(Map args, Object persistence, CacheRegenerator regenerator) { + super.init(args, regenerator); + String str = args.get(MAX_SIZE_PARAM); + maxSize = (str == null) ? 100_000 : Integer.parseInt(str); + str = args.get(MAX_RAM_MB_PARAM); + maxRamMB = (str == null) ? -1 : Integer.parseInt(str); + str = args.get(INITIAL_SIZE_PARAM); + initialSize = Math.min((str == null) ? maxSize : Integer.parseInt(str), maxSize); + str = args.get("initialRamMB"); + initialRamMB = Math.min((str == null) ? maxRamMB : Integer.parseInt(str), maxRamMB); + str = args.get(MAX_IDLE_TIME_PARAM); + int maxIdleTimeSec = -1; + if (str != null) { + maxIdleTimeSec = Integer.parseInt(str); + } + str = args.get("expireAfterWriteSeconds"); + int expireAfterWriteSeconds = -1; + if (str != null) { + expireAfterWriteSeconds = Integer.parseInt(str); + } + mqCache = buildCache(maxIdleTimeSec, expireAfterWriteSeconds); + return persistence; + } + + private Cache buildCache( + int maxIdleTimeSec, int expireAfterWriteSeconds) { + Caffeine builder = + Caffeine.newBuilder().initialCapacity(initialSize).removalListener(this).recordStats(); + if (maxIdleTimeSec > 0) { + builder.expireAfterAccess(Duration.ofSeconds(maxIdleTimeSec)); + } else if (expireAfterWriteSeconds > 0) { + builder.expireAfterWrite(Duration.ofSeconds(expireAfterWriteSeconds)); + } + + if (maxRamMB >= 0) { + builder.maximumWeight(maxRamMB * 1024L * 1024L); + builder.weigher( + (k, v) -> + (int) + (RamUsageEstimator.sizeOf(k) + + RamUsageEstimator.sizeOf(v.entry.getMatchQuery()))); + } else { + builder.maximumSize(maxSize); + } + return builder.build(); + } + + @Override + public int size() { + return mqCacheMap().size(); + } + + @Override + public VersionedQueryCacheEntry put(String key, VersionedQueryCacheEntry value) { + return mqCacheMap().put(key, value); + } + + @Override + public VersionedQueryCacheEntry get(String key) { + return mqCacheMap().get(key); + } + + @Override + public VersionedQueryCacheEntry remove(String key) { + return mqCacheMap().remove(key); + } + + @Override + public VersionedQueryCacheEntry computeIfAbsent( + String key, IOFunction mappingFunction) + throws IOException { + return mqCacheMap() + .computeIfAbsent( + key, + _key -> { + try { + return mappingFunction.apply(_key); + } catch (IOException e) { + throw new SolrException(ErrorCode.INVALID_STATE, "Could not update cache", e); + } + }); + } + + @Override + public void clear() { + mqCacheMap().clear(); + } + + @Override + public void warm(SolrIndexSearcher searcher, SolrCache old) { + try { + DefaultSavedSearchCache oldDefaultSavedSearchCache = + old instanceof DefaultSavedSearchCache ? (DefaultSavedSearchCache) old : null; + if (oldDefaultSavedSearchCache != null) { + mqCache = oldDefaultSavedSearchCache.mqCache; + maxVersionMap = oldDefaultSavedSearchCache.maxVersionMap; + } + if (regenerator != null) { + long nanoStart = System.nanoTime(); + regenerator.regenerateItem(searcher, this, old, null, null); + generationTimeMs = NANOSECONDS.toMillis(System.nanoTime() - nanoStart); + } + termFilter = new QueryTermFilterVisitor(searcher.getIndexReader()); + if (oldDefaultSavedSearchCache != null) { + var oldStats = oldDefaultSavedSearchCache.currentStats.get(); + priorHits = oldDefaultSavedSearchCache.priorHits + oldStats.hits; + priorLookups = oldDefaultSavedSearchCache.priorLookups + oldStats.lookups; + cumulativeGenerationTimeMs = + generationTimeMs + oldDefaultSavedSearchCache.cumulativeGenerationTimeMs; + cumulativeDocVisits = oldDefaultSavedSearchCache.cumulativeDocVisits + docVisits; + } + } catch (IOException e) { + throw new SolrException(ErrorCode.INVALID_STATE, "could not boostrap cache", e); + } + } + + @Override + public int getMaxSize() { + return maxSize; + } + + @Override + public void setMaxSize(int maxSize) { + throw new UnsupportedOperationException("maxSize is unsupported"); + } + + @Override + public int getMaxRamMB() { + return maxRamMB; + } + + @Override + public void setMaxRamMB(int maxRamMB) { + throw new UnsupportedOperationException("cannot set max RAM Mb"); + } + + @Override + public String getName() { + return name(); + } + + @Override + public String getDescription() { + return "Solr monitor query cache"; + } + + @Override + public void initializeMetrics(SolrMetricsContext parentContext, String scope) { + solrMetricsContext = parentContext.getChildContext(this); + var cacheMap = + new MetricsMap( + map -> { + var stats = currentStats.get(); + map.put(LOOKUPS_PARAM, stats.lookups); + map.put(HITS_PARAM, stats.hits); + map.put(HIT_RATIO_PARAM, rate(stats.hits, stats.lookups)); + map.put(SIZE_PARAM, size()); + map.put(MAX_RAM_MB_PARAM, getMaxRamMB()); + map.put(MAX_SIZE_PARAM, getMaxSize()); + map.put("generation_time", generationTimeMs); + map.put("doc_visits", docVisits); + long cumulativeLookups = priorLookups + stats.lookups; + long cumulativeHits = priorHits + stats.hits; + map.put("cumulative_lookups", cumulativeLookups); + map.put("cumulative_hits", cumulativeHits); + map.put("cumulative_generation_time", cumulativeGenerationTimeMs); + map.put("cumulative_hitratio", rate(cumulativeHits, cumulativeLookups)); + map.put( + "cumulative_generation_overhead", + rate(cumulativeGenerationTimeMs, cumulativeHits)); + map.put("cumulative_doc_visits", cumulativeDocVisits); + }); + solrMetricsContext.gauge(cacheMap, true, scope, getCategory().toString()); + } + + @Override + public void initialSearcher(SolrIndexSearcher initialSearcher) { + warm(initialSearcher, this); + } + + public int getInitialSize() { + return initialSize; + } + + private static double rate(long num, long den) { + return den == 0 ? 1.0 : (double) num / den; + } + + @Override + public SolrMetricsContext getSolrMetricsContext() { + return solrMetricsContext; + } + + private VersionedQueryCacheEntry compute( + String cacheId, + VersionedQueryCacheEntry prevEntry, + SavedSearchDataValues dataValues, + SavedSearchDecoder decoder) { + try { + var version = dataValues.getVersion(); + if (prevEntry == null || version > prevEntry.version) { + QueryDisjunct component = decoder.getDisjunct(dataValues, cacheId); + currentStats.updateAndGet(CurrentStats::miss); + return new VersionedQueryCacheEntry(component, version); + } + if (version == prevEntry.version) { + currentStats.updateAndGet(CurrentStats::hit); + } else { + currentStats.updateAndGet(CurrentStats::miss); + } + return prevEntry; + } catch (Exception e) { + throw new SolrException(ErrorCode.INVALID_STATE, "Failed to update SavedSearchCache", e); + } + } + + @Override + public void onRemoval(String key, VersionedQueryCacheEntry value, RemovalCause cause) { + /* TODO revisit */ + } + + private static final class CurrentStats { + + private final long hits; + private final long lookups; + + private CurrentStats(long hits, long lookups) { + this.hits = hits; + this.lookups = lookups; + } + + private static CurrentStats init() { + return new CurrentStats(0, 0); + } + + private static CurrentStats hit(CurrentStats old) { + return new CurrentStats(old.hits + 1, old.lookups + 1); + } + + private static CurrentStats miss(CurrentStats old) { + return new CurrentStats(old.hits, old.lookups + 1); + } + } + + public static class LatestRegenerator implements CacheRegenerator { + + private static final int MAX_BATCH_SIZE = 1 << 10; + private static final Sort VERSION_SORT = + new Sort(new SortField(CommonParams.VERSION_FIELD, SortField.Type.LONG)); + + @Override + public boolean regenerateItem( + SolrIndexSearcher searcher, + SolrCache newCache, + SolrCache oldCache, + K oldKey, + V oldVal) + throws IOException { + if (!(newCache instanceof DefaultSavedSearchCache)) { + throw new IllegalArgumentException( + this.getClass().getSimpleName() + + " only supports " + + DefaultSavedSearchCache.class.getSimpleName()); + } + var cache = (DefaultSavedSearchCache) newCache; + var reader = searcher.getIndexReader(); + int batchSize = Math.min(MAX_BATCH_SIZE, cache.getInitialSize()); + boolean isDone = false; + int countBasedBatches = cache.getInitialSize() / batchSize; + long targetInitialRam = ((DefaultSavedSearchCache) newCache).initialRamMB * 1024L * 1024L; + long estimatedWeight = 0; + SolrCore core = searcher.getCore(); + SavedSearchDecoder decoder = new SavedSearchDecoder(core); + for (LeafReaderContext ctx : reader.leaves()) { + LeafReader leafReader = ctx.reader(); + CacheKey cacheKey = leafReader.getCoreCacheHelper().getKey(); + long maxLeafVersion = cache.maxVersionMap.get(cacheKey, __ -> -1L); + var topDocs = + new IndexSearcher(leafReader) + .search(versionRangeQuery(maxLeafVersion), batchSize, VERSION_SORT) + .scoreDocs; + Arrays.sort(topDocs, (a, b) -> Long.compare(b.doc, a.doc)); + while (topDocs.length > 0 && !isDone) { + SavedSearchDataValues dataValues = + new SavedSearchDataValues(ctx, core.getLatestSchema().getUniqueKeyField().getName()); + for (ScoreDoc topDoc : topDocs) { + if (dataValues.advanceTo(topDoc.doc)) { + var cacheEntry = cache.computeIfStale(dataValues, decoder); + estimatedWeight += RamUsageEstimator.sizeOf(cacheEntry.entry.getMatchQuery()); + estimatedWeight += RamUsageEstimator.sizeOf(cacheEntry.entry.getId()); + long version = dataValues.getVersion(); + maxLeafVersion = Math.max(maxLeafVersion, version); + cache.docVisits++; + } + } + topDocs = + new IndexSearcher(leafReader) + .search(versionRangeQuery(maxLeafVersion + 1), batchSize, VERSION_SORT) + .scoreDocs; + Arrays.sort(topDocs, (a, b) -> Long.compare(b.doc, a.doc)); + if (targetInitialRam > 0L) { + isDone = estimatedWeight >= targetInitialRam; + } else { + isDone = --countBasedBatches <= 0; + } + } + cache.maxVersionMap.put(cacheKey, maxLeafVersion); + } + return false; + } + + private static Query versionRangeQuery(long version) { + return LongPoint.newRangeQuery(CommonParams.VERSION_FIELD, version, Long.MAX_VALUE); + } + } +} diff --git a/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/cache/SavedSearchCache.java b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/cache/SavedSearchCache.java new file mode 100644 index 000000000000..31ef8e9aaf2e --- /dev/null +++ b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/cache/SavedSearchCache.java @@ -0,0 +1,45 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.solr.savedsearch.cache; + +import java.io.IOException; +import java.util.function.BiPredicate; +import org.apache.lucene.util.BytesRef; +import org.apache.solr.savedsearch.SavedSearchDataValues; +import org.apache.solr.savedsearch.SavedSearchDecoder; + +public interface SavedSearchCache { + + VersionedQueryCacheEntry computeIfStale( + SavedSearchDataValues dataValues, SavedSearchDecoder decoder) throws IOException; + + BiPredicate termFilter(); + + class VersionedQueryCacheEntry { + + public final SavedSearchDataValues.QueryDisjunct entry; + public final long version; + + VersionedQueryCacheEntry(SavedSearchDataValues.QueryDisjunct entry, long version) { + this.entry = entry; + this.version = version; + } + } +} diff --git a/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/cache/package-info.java b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/cache/package-info.java new file mode 100644 index 000000000000..a9afe9f75386 --- /dev/null +++ b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/cache/package-info.java @@ -0,0 +1,21 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +/** This package contains caching logic for Solr's lucene-monitor integration. */ +package org.apache.solr.savedsearch.cache; diff --git a/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/package-info.java b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/package-info.java new file mode 100644 index 000000000000..77d399ca08db --- /dev/null +++ b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/package-info.java @@ -0,0 +1,21 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +/** This package contains Solr's lucene monitor integration. */ +package org.apache.solr.savedsearch; diff --git a/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/search/PresearcherFactory.java b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/search/PresearcherFactory.java new file mode 100644 index 000000000000..0452b891537b --- /dev/null +++ b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/search/PresearcherFactory.java @@ -0,0 +1,145 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.solr.savedsearch.search; + +import java.lang.reflect.Constructor; +import java.util.List; +import java.util.Set; +import org.apache.lucene.monitor.CustomQueryHandler; +import org.apache.lucene.monitor.MultipassTermFilteredPresearcher; +import org.apache.lucene.monitor.Presearcher; +import org.apache.lucene.monitor.TermFilteredPresearcher; +import org.apache.lucene.monitor.TermWeightor; +import org.apache.lucene.util.ResourceLoader; +import org.apache.solr.savedsearch.AliasingPresearcher; + +public class PresearcherFactory { + + // The default prefix is long because we don't want it to clash + // with a user-defined name pattern and the longest one wins. + // {@link + // https://cwiki.apache.org/confluence/display/solr/SchemaXml#Dynamic_fields:~:text=Longer%20patterns%20will%20be%20matched%20first} + // In the worst case this can be overridden by the user but ideally this never comes up. + public static final String DEFAULT_ALIAS_PREFIX = + "_________________________________________________saved_search_alias_"; + + public static final String TERM_FILTERED = TermFilteredPresearcher.class.getSimpleName(); + public static final String MULTI_PASS_TERM_FILTERED = + MultipassTermFilteredPresearcher.class.getSimpleName(); + + private PresearcherFactory() {} + + static Presearcher build( + ResourceLoader resourceLoader, PresearcherParameters presearcherParameters) { + Presearcher presearcher; + String type = presearcherParameters.presearcherType; + if (TERM_FILTERED.equals(type) || MULTI_PASS_TERM_FILTERED.equals(type)) { + TermWeightor weightor = + constructZeroArgClass( + resourceLoader, + TermWeightor.class, + presearcherParameters.termWeightorType, + TermFilteredPresearcher.DEFAULT_WEIGHTOR); + List customQueryHandlers = List.of(); + Set filterFields = Set.of(); + if (TERM_FILTERED.equals(type)) { + if (presearcherParameters.applyFieldNameAlias) { + presearcher = + AliasingPresearcher.TermFiltered.build( + weightor, customQueryHandlers, filterFields, presearcherParameters.aliasPrefix); + } else { + presearcher = new TermFilteredPresearcher(weightor, customQueryHandlers, filterFields); + } + } else { + if (presearcherParameters.applyFieldNameAlias) { + presearcher = + AliasingPresearcher.MultiPassTermFiltered.build( + presearcherParameters.numberOfPasses, + presearcherParameters.minWeight, + weightor, + customQueryHandlers, + filterFields, + presearcherParameters.aliasPrefix); + } else { + presearcher = + new MultipassTermFilteredPresearcher( + presearcherParameters.numberOfPasses, + presearcherParameters.minWeight, + weightor, + customQueryHandlers, + filterFields); + } + } + } else { + presearcher = + constructZeroArgClass(resourceLoader, Presearcher.class, type, Presearcher.NO_FILTERING); + } + return presearcher; + } + + private static T constructZeroArgClass( + ResourceLoader loader, Class expectedType, String typeName, T defaultVal) { + if (typeName == null) { + return defaultVal; + } + try { + Class implClass = loader.findClass(typeName, expectedType); + Constructor c = implClass.getConstructor(); + return c.newInstance(); + } catch (Exception e) { + throw new IllegalStateException( + "Could not construct custom " + expectedType.getSimpleName() + " of type " + typeName, e); + } + } + + public static class PresearcherParameters { + + private String presearcherType; + private String termWeightorType; + private boolean applyFieldNameAlias = false; + private int numberOfPasses = 0; + private float minWeight = 0; + private String aliasPrefix = DEFAULT_ALIAS_PREFIX; + + public void setPresearcherType(String presearcherType) { + this.presearcherType = presearcherType; + } + + public void setTermWeightorType(String termWeightorType) { + this.termWeightorType = termWeightorType; + } + + public void setApplyFieldNameAlias(boolean applyFieldNameAlias) { + this.applyFieldNameAlias = applyFieldNameAlias; + } + + public void setNumberOfPasses(int numberOfPasses) { + this.numberOfPasses = numberOfPasses; + } + + public void setMinWeight(float minWeight) { + this.minWeight = minWeight; + } + + public void setAliasPrefix(String aliasPrefix) { + this.aliasPrefix = aliasPrefix; + } + } +} diff --git a/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/search/ReverseSearchComponent.java b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/search/ReverseSearchComponent.java new file mode 100644 index 000000000000..a30e3f981880 --- /dev/null +++ b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/search/ReverseSearchComponent.java @@ -0,0 +1,283 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.solr.savedsearch.search; + +import static org.apache.solr.savedsearch.search.PresearcherFactory.DEFAULT_ALIAS_PREFIX; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.BiPredicate; +import java.util.function.Function; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.monitor.CandidateMatcher; +import org.apache.lucene.monitor.Presearcher; +import org.apache.lucene.monitor.QueryDecomposer; +import org.apache.lucene.monitor.QueryMatch; +import org.apache.lucene.monitor.TermFilteredPresearcher; +import org.apache.lucene.monitor.Visitors.DocumentBatchVisitor; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.component.DebugComponent; +import org.apache.solr.handler.component.QueryComponent; +import org.apache.solr.handler.component.ResponseBuilder; +import org.apache.solr.handler.loader.JsonLoader; +import org.apache.solr.savedsearch.AliasingPresearcher; +import org.apache.solr.savedsearch.SavedSearchDataValues; +import org.apache.solr.savedsearch.SavedSearchDecoder; +import org.apache.solr.savedsearch.cache.DefaultSavedSearchCache; +import org.apache.solr.savedsearch.cache.SavedSearchCache; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.update.DocumentBuilder; +import org.apache.solr.util.SolrPluginUtils; +import org.apache.solr.util.plugin.SolrCoreAware; + +public class ReverseSearchComponent extends QueryComponent implements SolrCoreAware { + + public static final String COMPONENT_NAME = "reverseSearch"; + + private static final String SAVED_SEARCH_CACHE_NAME_KEY = "savedSearchCacheName"; + private static final String SAVED_SEARCH_CACHE_NAME_DEFAULT = "savedSearchCache"; + private static final String REVERSE_SEARCH_DOCUMENTS_KEY = "reverseSearchDocuments"; + private String savedSearchCacheName = SAVED_SEARCH_CACHE_NAME_DEFAULT; + + private QueryDecomposer queryDecomposer; + private Presearcher presearcher; + private PresearcherFactory.PresearcherParameters presearcherParameters; + + @Override + public void init(NamedList args) { + super.init(args); + Object savedSearchCacheName = args.remove(SAVED_SEARCH_CACHE_NAME_KEY); + if (savedSearchCacheName != null) { + this.savedSearchCacheName = (String) savedSearchCacheName; + } + presearcherParameters = new PresearcherFactory.PresearcherParameters(); + SolrPluginUtils.invokeSetters(presearcherParameters, args); + } + + @Override + public void process(ResponseBuilder rb) throws IOException { + try (final DocumentBatchVisitor documentBatchVisitor = + documentBatchVisitor(rb.req.getJSON(), rb.req.getSchema())) { + if (documentBatchVisitor == null) { + super.process(rb); + return; + } + final LeafReader documentBatch = documentBatchVisitor.get(); + final SavedSearchCache savedSearchCache = + (DefaultSavedSearchCache) rb.req.getSearcher().getCache(this.savedSearchCacheName); + + final BiPredicate termAcceptor = + Optional.ofNullable(savedSearchCache) + .map(SavedSearchCache::termFilter) + .orElse((__, ___) -> true); + final Query preFilterQuery = presearcher.buildQuery(documentBatch, termAcceptor); + final SolrMatcherSink solrMatcherSink = + new DefaultSolrMatcherSink<>( + QueryMatch.SIMPLE_MATCHER::createMatcher, new IndexSearcher(documentBatch)); + rb.setQuery(reverseSearchQuery(preFilterQuery, rb, solrMatcherSink, savedSearchCache)); + super.process(rb); + if (rb.isDebug()) { + ReverseSearchQuery.Metadata metadata = solrMatcherSink.getMetadata(); + DebugComponent.CustomDebugInfoSources debugInfoSources = + (DebugComponent.CustomDebugInfoSources) + rb.req + .getContext() + .computeIfAbsent( + DebugComponent.CustomDebugInfoSources.KEY, + key -> new DebugComponent.CustomDebugInfoSources()); + SimpleOrderedMap info = new SimpleOrderedMap<>(); + info.add("queriesRun", metadata.getQueriesRun()); + debugInfoSources.add( + new DebugComponent.CustomDebugInfoSource("reverse-search-debug", info)); + } + } + } + + private static ReverseSearchQuery reverseSearchQuery( + Query preFilterQuery, + ResponseBuilder rb, + SolrMatcherSink solrMatcherSink, + SavedSearchCache savedSearchCache) { + final SavedSearchDecoder savedSearchDecoder = new SavedSearchDecoder(rb.req.getCore()); + final ReverseSearchQuery.ReverseSearchContext context = + new ReverseSearchQuery.ReverseSearchContext( + savedSearchCache, + savedSearchDecoder, + solrMatcherSink, + rb.req.getSchema().getUniqueKeyField().getName()); + + return new ReverseSearchQuery(context, preFilterQuery); + } + + @SuppressWarnings({"unchecked"}) + private static DocumentBatchVisitor documentBatchVisitor( + Map json, IndexSchema indexSchema) { + if (json == null || !json.containsKey("params")) { + return null; + } + Object jsonParams = json.get("params"); + if (!(jsonParams instanceof Map)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "need params"); + } + Map paramMap = (Map) jsonParams; + Object documents = paramMap.get(REVERSE_SEARCH_DOCUMENTS_KEY); + if (!(documents instanceof List)) { + return null; + } + List luceneDocs = new ArrayList<>(); + for (Object document : (List) documents) { + if (!(document instanceof Map) + || !((Map) document).keySet().stream().allMatch(key -> key instanceof String)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "document needs to be a string-keyed map"); + } + Map docAsMap = (Map) document; + docAsMap.putIfAbsent(indexSchema.getUniqueKeyField().getName(), UUID.randomUUID().toString()); + var solrInputDoc = JsonLoader.buildDoc((Map) document); + var luceneDoc = DocumentBuilder.toDocument(solrInputDoc, indexSchema); + luceneDocs.add(luceneDoc); + } + return DocumentBatchVisitor.of(indexSchema.getIndexAnalyzer(), luceneDocs); + } + + @Override + public String getDescription() { + return "Component that integrates with lucene monitor for reverse search."; + } + + @Override + public void inform(SolrCore core) { + queryDecomposer = new QueryDecomposer(); + presearcher = PresearcherFactory.build(core.getResourceLoader(), presearcherParameters); + var schema = core.getLatestSchema(); + if (presearcher instanceof AliasingPresearcher) { + String prefix = ((AliasingPresearcher) presearcher).getPrefix() + "*"; + String maxLengthPattern = + Arrays.stream(schema.getDynamicFieldPrototypes()) + .map(SchemaField::getName) + .max(Comparator.comparingInt(String::length)) + .orElse(""); + if (!maxLengthPattern.equals(prefix) && prefix.length() <= maxLengthPattern.length()) { + throw new IllegalStateException( + "Dynamic field pattern " + + maxLengthPattern + + " is incompatible with presearcher alias pattern " + + prefix + + ". Increase the length of presearcher alias prefix to be at least" + + (maxLengthPattern.length() + 1) + + " characters, i.e. set aliasPrefix to " + + "_".repeat(maxLengthPattern.length() - DEFAULT_ALIAS_PREFIX.length() + 1) + + DEFAULT_ALIAS_PREFIX + + " and update the schema accordingly. Refer to documentation on overriding aliasPrefix in " + + this.getClass().getSimpleName()); + } + var fieldType = schema.getDynamicFieldType(prefix); + if (!fieldType.isTokenized() || !fieldType.isMultiValued()) { + throw new IllegalStateException( + "presearcher-aliased field must be tokenized and multi-valued."); + } + if (!fieldType.isTokenized() || !fieldType.isMultiValued()) { + throw new IllegalStateException( + "presearcher-aliased field must be tokenized and multi-valued."); + } + } + + for (String fieldName : SavedSearchDataValues.REQUIRED_MONITOR_SCHEMA_FIELDS) { + var field = schema.getFieldOrNull(fieldName); + if (field == null) { + throw new IllegalStateException( + fieldName + " must be defined in schema for saved search to work."); + } + } + + if (presearcher instanceof TermFilteredPresearcher + || presearcher instanceof AliasingPresearcher) { + var anyTokenField = schema.getFieldOrNull(TermFilteredPresearcher.ANYTOKEN_FIELD); + if (anyTokenField == null || !anyTokenField.tokenized() || !anyTokenField.multiValued()) { + throw new IllegalStateException( + TermFilteredPresearcher.ANYTOKEN_FIELD + " field must be tokenized and multi-valued."); + } + } + } + + public QueryDecomposer getQueryDecomposer() { + return queryDecomposer; + } + + public Presearcher getPresearcher() { + return presearcher; + } + + static class DefaultSolrMatcherSink implements SolrMatcherSink { + + private final Function> matcherFactory; + private final IndexSearcher docBatchSearcher; + private final ConcurrentHashMap.KeySetView metadataSet = + ConcurrentHashMap.newKeySet(); + + DefaultSolrMatcherSink( + Function> matcherFactory, + IndexSearcher docBatchSearcher) { + this.matcherFactory = matcherFactory; + this.docBatchSearcher = docBatchSearcher; + } + + @Override + public boolean matchQuery(String queryId, Query matchQuery, Map metadata) + throws IOException { + var matcher = matcherFactory.apply(docBatchSearcher); + matcher.matchQuery(queryId, matchQuery, metadata); + var matches = matcher.finish(Long.MIN_VALUE, 1); + for (int doc = 0; doc < matches.getBatchSize(); doc++) { + var match = matches.getMatches(doc); + if (!match.isEmpty()) { + return true; + } + } + return false; + } + + @Override + public void captureMetadata(ReverseSearchQuery.Metadata metadata) { + metadataSet.add(metadata); + } + + @Override + public ReverseSearchQuery.Metadata getMetadata() { + return metadataSet.stream() + .reduce(ReverseSearchQuery.Metadata.IDENTITY, ReverseSearchQuery.Metadata::add); + } + } +} diff --git a/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/search/ReverseSearchHandler.java b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/search/ReverseSearchHandler.java new file mode 100644 index 000000000000..1b6186ec3d9d --- /dev/null +++ b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/search/ReverseSearchHandler.java @@ -0,0 +1,36 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.solr.savedsearch.search; + +import java.util.ArrayList; +import java.util.List; +import org.apache.solr.handler.component.DebugComponent; +import org.apache.solr.handler.component.SearchHandler; + +public class ReverseSearchHandler extends SearchHandler { + + @Override + protected List getDefaultComponents() { + ArrayList names = new ArrayList<>(1); + names.add(ReverseSearchComponent.COMPONENT_NAME); + names.add(DebugComponent.COMPONENT_NAME); + return names; + } +} diff --git a/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/search/ReverseSearchQuery.java b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/search/ReverseSearchQuery.java new file mode 100644 index 000000000000..292fddb3d069 --- /dev/null +++ b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/search/ReverseSearchQuery.java @@ -0,0 +1,290 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.solr.savedsearch.search; + +import java.io.IOException; +import java.util.Collection; +import java.util.Objects; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.solr.savedsearch.SavedSearchDataValues; +import org.apache.solr.savedsearch.SavedSearchDataValues.QueryDisjunct; +import org.apache.solr.savedsearch.SavedSearchDecoder; +import org.apache.solr.savedsearch.cache.SavedSearchCache; + +class ReverseSearchQuery extends Query { + + private final ReverseSearchContext reverseSearchContext; + private final Query presearchQuery; + + public ReverseSearchQuery(ReverseSearchContext reverseSearchContext, Query presearchQuery) { + this.reverseSearchContext = reverseSearchContext; + this.presearchQuery = presearchQuery; + } + + @Override + public void visit(QueryVisitor visitor) { + presearchQuery.visit(visitor); + } + + @Override + public boolean equals(Object o) { + return this == o; + } + + @Override + public Query rewrite(IndexSearcher indexSearcher) throws IOException { + Query rewritten = presearchQuery.rewrite(indexSearcher); + if (!Objects.equals(rewritten, presearchQuery)) { + return new ReverseSearchQuery(reverseSearchContext, presearchQuery.rewrite(indexSearcher)); + } + return super.rewrite(indexSearcher); + } + + @Override + public int hashCode() { + return System.identityHashCode(this); + } + + @Override + public String toString(String field) { + return this.getClass().getSimpleName() + + "(presearchQuery=" + + presearchQuery.toString(field) + + ")"; + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) + throws IOException { + return new ReverseSearchWeight( + this, reverseSearchContext, presearchQuery.createWeight(searcher, scoreMode, boost)); + } + + static class ReverseSearchWeight extends Weight { + + private final ReverseSearchContext reverseSearchContext; + private final Weight presearchWeight; + + ReverseSearchWeight( + Query query, ReverseSearchContext reverseSearchContext, Weight presearchWeight) { + super(query); + this.reverseSearchContext = reverseSearchContext; + this.presearchWeight = presearchWeight; + } + + @Override + public Explanation explain(LeafReaderContext context, int doc) throws IOException { + return presearchWeight.explain(context, doc); + } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return false; + } + + @Override + public int count(LeafReaderContext context) throws IOException { + return presearchWeight.count(context); + } + + @Override + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + Scorer scorer = presearchWeight.scorer(context); + if (scorer == null) { + return null; + } + Scorer wrappedScorer = new ReverseSearchScorer(scorer, reverseSearchContext, context); + return new ScorerSupplier() { + + @Override + public Scorer get(long leadCost) { + return wrappedScorer; + } + + @Override + public long cost() { + return Long.MAX_VALUE; + } + }; + } + + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return presearchWeight.matches(context, doc); + } + } + + static class ReverseSearchScorer extends Scorer { + private static final float MATCH_COST = 100.0f; + + private final Scorer presearchScorer; + private final SavedSearchCache savedSearchCache; + private final SavedSearchDecoder queryDecoder; + private final SolrMatcherSink matcherSink; + private final Metadata metadata; + private final SavedSearchDataValues dataValues; + + ReverseSearchScorer( + Scorer presearchScorer, + ReverseSearchContext reverseSearchContext, + LeafReaderContext leafReaderContext) + throws IOException { + this.presearchScorer = presearchScorer; + this.savedSearchCache = reverseSearchContext.queryCache; + this.queryDecoder = reverseSearchContext.queryDecoder; + this.matcherSink = reverseSearchContext.solrMatcherSink; + this.metadata = new Metadata(); + matcherSink.captureMetadata(metadata); + this.dataValues = + new SavedSearchDataValues(leafReaderContext, reverseSearchContext.idFieldName); + } + + @Override + public DocIdSetIterator iterator() { + return presearchScorer.iterator(); + } + + @Override + public float getMaxScore(int upTo) throws IOException { + return presearchScorer.getMaxScore(upTo); + } + + @Override + public float score() throws IOException { + return presearchScorer.score(); + } + + @Override + public int docID() { + return presearchScorer.docID(); + } + + @Override + public int advanceShallow(int target) throws IOException { + return presearchScorer.advanceShallow(target); + } + + @Override + public float smoothingScore(int docId) throws IOException { + return presearchScorer.smoothingScore(docId); + } + + @Override + public void setMinCompetitiveScore(float minScore) throws IOException { + presearchScorer.setMinCompetitiveScore(minScore); + } + + @Override + public Collection getChildren() throws IOException { + return presearchScorer.getChildren(); + } + + @Override + public TwoPhaseIterator twoPhaseIterator() { + + return new TwoPhaseIterator(presearchScorer.iterator()) { + + private String lastQueryId; + + @Override + public boolean matches() throws IOException { + metadata.queriesRun++; + int doc = approximation.docID(); + dataValues.advanceTo(doc); + var entry = getEntry(dataValues); + var queryId = dataValues.getQueryId(); + var originalMatchQuery = entry.getMatchQuery(); + + var matchQuery = new ConstantScoreQuery(originalMatchQuery); + + boolean isMatch = matcherSink.matchQuery(queryId, matchQuery, entry.getMetadata()); + if (isMatch && !queryId.equals(lastQueryId)) { + lastQueryId = queryId; + return true; + } + return false; + } + + private QueryDisjunct getEntry(SavedSearchDataValues dataValues) throws IOException { + var versionedEntry = + savedSearchCache == null + ? null + : savedSearchCache.computeIfStale(dataValues, queryDecoder); + return (versionedEntry == null || versionedEntry.version != dataValues.getVersion()) + ? queryDecoder.getDisjunct(dataValues, dataValues.getCacheId()) + : versionedEntry.entry; + } + + @Override + public float matchCost() { + return MATCH_COST; + } + }; + } + } + + static class ReverseSearchContext { + + private final SavedSearchCache queryCache; + private final SavedSearchDecoder queryDecoder; + private final SolrMatcherSink solrMatcherSink; + private final String idFieldName; + + ReverseSearchContext( + SavedSearchCache queryCache, + SavedSearchDecoder queryDecoder, + SolrMatcherSink solrMatcherSink, + String idFieldName) { + this.queryCache = queryCache; + this.queryDecoder = queryDecoder; + this.solrMatcherSink = solrMatcherSink; + this.idFieldName = idFieldName; + } + } + + static class Metadata { + + static final Metadata IDENTITY = new Metadata(); + + private int queriesRun; + + int getQueriesRun() { + return queriesRun; + } + + Metadata add(Metadata metadata) { + Metadata result = new Metadata(); + result.queriesRun = this.queriesRun + metadata.queriesRun; + return result; + } + } +} diff --git a/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/search/SolrMatcherSink.java b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/search/SolrMatcherSink.java new file mode 100644 index 000000000000..1fd2a1c9fd1c --- /dev/null +++ b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/search/SolrMatcherSink.java @@ -0,0 +1,34 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.solr.savedsearch.search; + +import java.io.IOException; +import java.util.Map; +import org.apache.lucene.search.Query; + +interface SolrMatcherSink { + + boolean matchQuery(String queryId, Query matchQuery, Map metadata) + throws IOException; + + void captureMetadata(ReverseSearchQuery.Metadata metadata); + + ReverseSearchQuery.Metadata getMetadata(); +} diff --git a/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/search/package-info.java b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/search/package-info.java new file mode 100644 index 000000000000..0fd177d01376 --- /dev/null +++ b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/search/package-info.java @@ -0,0 +1,21 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +/** This package contains search logic for Solr's lucene-monitor integration. */ +package org.apache.solr.savedsearch.search; diff --git a/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/update/SavedSearchUpdateProcessorFactory.java b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/update/SavedSearchUpdateProcessorFactory.java new file mode 100644 index 000000000000..29757a34965d --- /dev/null +++ b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/update/SavedSearchUpdateProcessorFactory.java @@ -0,0 +1,272 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.solr.savedsearch.update; + +import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.InvertableType; +import org.apache.lucene.document.StoredValue; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.IndexableFieldType; +import org.apache.lucene.monitor.MonitorQuery; +import org.apache.lucene.monitor.Presearcher; +import org.apache.lucene.monitor.QueryDecomposer; +import org.apache.lucene.util.BytesRef; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.util.JavaBinCodec; +import org.apache.solr.core.SolrCore; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.savedsearch.SavedSearchDataValues; +import org.apache.solr.savedsearch.SavedSearchDataValues.QueryDisjunct; +import org.apache.solr.savedsearch.SimpleQueryParser; +import org.apache.solr.savedsearch.search.ReverseSearchComponent; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.update.AddUpdateCommand; +import org.apache.solr.update.processor.UpdateRequestProcessor; +import org.apache.solr.update.processor.UpdateRequestProcessorFactory; +import org.apache.solr.util.plugin.SolrCoreAware; + +public class SavedSearchUpdateProcessorFactory extends UpdateRequestProcessorFactory + implements SolrCoreAware { + + private QueryDecomposer queryDecomposer; + private Presearcher presearcher; + + @Override + public UpdateRequestProcessor getInstance( + SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) { + return new SavedSearchUpdateRequestProcessor(next, req.getCore(), queryDecomposer, presearcher); + } + + @Override + public void inform(SolrCore core) { + ReverseSearchComponent rsc = + (ReverseSearchComponent) + core.getSearchComponents().get(ReverseSearchComponent.COMPONENT_NAME); + presearcher = rsc.getPresearcher(); + queryDecomposer = rsc.getQueryDecomposer(); + } + + static class SavedSearchUpdateRequestProcessor extends UpdateRequestProcessor { + + private final SolrCore core; + private final IndexSchema indexSchema; + private final QueryDecomposer queryDecomposer; + private final Presearcher presearcher; + private final Set allowedFieldNames; + private final SchemaField queryIdField; + private final SchemaField monitorQueryField; + + public SavedSearchUpdateRequestProcessor( + UpdateRequestProcessor next, + SolrCore core, + QueryDecomposer queryDecomposer, + Presearcher presearcher) { + super(next); + this.core = core; + this.indexSchema = core.getLatestSchema(); + this.queryDecomposer = queryDecomposer; + this.presearcher = presearcher; + this.queryIdField = indexSchema.getField(SavedSearchDataValues.QUERY_ID); + this.monitorQueryField = indexSchema.getField(SavedSearchDataValues.MONITOR_QUERY); + this.allowedFieldNames = + Set.of( + SavedSearchDataValues.MONITOR_QUERY, + indexSchema.getUniqueKeyField().getName(), + CommonParams.VERSION_FIELD); + } + + @Override + public void processAdd(AddUpdateCommand cmd) throws IOException { + var solrInputDocument = cmd.getSolrInputDocument(); + String idFieldName = indexSchema.getUniqueKeyField().getName(); + var queryId = (String) solrInputDocument.getFieldValue(idFieldName); + var queryFieldValue = solrInputDocument.getFieldValue(SavedSearchDataValues.MONITOR_QUERY); + if (queryFieldValue == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Document is missing mandatory " + + SavedSearchDataValues.MONITOR_QUERY + + " field which is required by " + + getClass().getSimpleName() + + "."); + } + List children = + Optional.of(queryFieldValue) + .filter(String.class::isInstance) + .map(String.class::cast) + .map( + queryStr -> + new MonitorQuery( + queryId, SimpleQueryParser.parse(queryStr, core), queryStr, Map.of())) + .stream() + .flatMap(this::decompose) + .map(this::toSolrInputDoc) + .collect(Collectors.toList()); + if (children.isEmpty()) { + throw new SolrException( + SolrException.ErrorCode.INVALID_STATE, "Query could not be decomposed"); + } + SolrInputDocument firstChild = children.get(0); + if (solrInputDocument.hasChildDocuments()) { + solrInputDocument.getChildDocuments().clear(); + } + solrInputDocument.addChildDocuments(children.stream().skip(1).collect(Collectors.toList())); + for (var firstChildField : firstChild) { + solrInputDocument.setField(firstChildField.getName(), firstChildField.getValue()); + } + super.processAdd(cmd); + } + + private Stream decompose(MonitorQuery monitorQuery) { + return QueryDisjunct.decompose(monitorQuery, queryDecomposer).stream() + .map( + disjunct -> { + Document doc = + presearcher.indexQuery(disjunct.getMatchQuery(), monitorQuery.getMetadata()); + doc.add(indexSchema.getUniqueKeyField().createField(disjunct.getId())); + doc.add(queryIdField.createField(disjunct.getQueryId())); + doc.add(monitorQueryField.createField(monitorQuery.getQueryString())); + return doc; + }); + } + + private SolrInputDocument toSolrInputDoc(Document subDocument) { + SolrInputDocument out = new SolrInputDocument(); + for (var f : subDocument.getFields()) { + Object existing = out.get(f.name()); + if (existing == null) { + SchemaField sf = indexSchema.getFieldOrNull(f.name()); + if (sf != null && sf.multiValued()) { + List vals = new ArrayList<>(); + if (f.fieldType().docValuesType() == DocValuesType.SORTED_NUMERIC) { + vals.add(sf.getType().toObject(f)); + } else { + vals.add(materialize(f)); + } + out.setField(f.name(), vals); + } else { + out.setField(f.name(), materialize(f)); + } + } else { + out.addField(f.name(), materialize(f)); + } + } + return out; + } + + private static Object materialize(IndexableField in) { + if (in instanceof Field && ((Field) in).tokenStreamValue() != null) { + return new DerivedSkipTLogField((Field) in); + } + if (in.numericValue() != null) { + return in.numericValue(); + } + if (in.binaryValue() != null) { + return in.binaryValue(); + } + if (in.stringValue() != null) { + return in.stringValue(); + } + throw new IllegalArgumentException("could not clone field " + in.name()); + } + + private static class DerivedSkipTLogField + implements IndexableField, JavaBinCodec.ObjectResolver { + + private final Field ref; + + public DerivedSkipTLogField(Field ref) { + this.ref = ref; + } + + @Override + public Object resolve(Object o, JavaBinCodec codec) { + return ""; + } + + @Override + public String name() { + return ref.name(); + } + + @Override + public IndexableFieldType fieldType() { + return ref.fieldType(); + } + + @Override + public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) { + return ref.tokenStream(analyzer, reuse); + } + + @Override + public BytesRef binaryValue() { + return ref.binaryValue(); + } + + @Override + public String stringValue() { + return ref.stringValue(); + } + + @Override + public CharSequence getCharSequenceValue() { + return ref.getCharSequenceValue(); + } + + @Override + public Reader readerValue() { + return ref.readerValue(); + } + + @Override + public Number numericValue() { + return ref.numericValue(); + } + + @Override + public StoredValue storedValue() { + return null; + } + + @Override + public InvertableType invertableType() { + return InvertableType.TOKEN_STREAM; + } + } + } +} diff --git a/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/update/package-info.java b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/update/package-info.java new file mode 100644 index 000000000000..e87f29ee76cc --- /dev/null +++ b/solr/modules/saved-search/src/java/org/apache/solr/savedsearch/update/package-info.java @@ -0,0 +1,21 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +/** This package contains update logic for Solr's lucene-monitor integration. */ +package org.apache.solr.savedsearch.update; diff --git a/solr/modules/saved-search/src/test-files/monitor/BigDisjunctionQuery.xml b/solr/modules/saved-search/src/test-files/monitor/BigDisjunctionQuery.xml new file mode 100644 index 000000000000..49d26cece00e --- /dev/null +++ b/solr/modules/saved-search/src/test-files/monitor/BigDisjunctionQuery.xml @@ -0,0 +1,1218 @@ + + + + + + + + + + TEST_DATA_ITEM_RD1PF93 + TEST_DATA_ITEM_RDSZ2T0 + TEST_DATA_ITEM_RJXPZR3 + TEST_DATA_ITEM_RKYPJR5 + TEST_DATA_ITEM_RM7XHH8 + TEST_DATA_ITEM_RNXF8T7 + TEST_DATA_ITEM_RP5HKT7 + TEST_DATA_ITEM_RP7FX08 + TEST_DATA_ITEM_RPWYNV0 + TEST_DATA_ITEM_RRGSHJ7 + TEST_DATA_ITEM_RVG4979 + TEST_DATA_ITEM_RZFGN37 + TEST_DATA_ITEM_S03JPK3 + TEST_DATA_ITEM_S1HHY24 + TEST_DATA_ITEM_S1LQXM5 + TEST_DATA_ITEM_S2K1884 + TEST_DATA_ITEM_S2L0213 + TEST_DATA_ITEM_S2L11V1 + TEST_DATA_ITEM_S2L1YW8 + TEST_DATA_ITEM_S42MY19 + TEST_DATA_ITEM_S7P0Q34 + TEST_DATA_ITEM_S7P5H84 + TEST_DATA_ITEM_SBXZ9J5 + TEST_DATA_ITEM_SL59VF9 + TEST_DATA_ITEM_SNC4234 + TEST_DATA_ITEM_SSQFBD5 + TEST_DATA_ITEM_T5JZ7Y8 + TEST_DATA_ITEM_T68GVF0 + TEST_DATA_ITEM_T8B8C35 + TEST_DATA_ITEM_THX1KC6 + TEST_DATA_ITEM_TJHY616 + TEST_DATA_ITEM_TJJ1191 + TEST_DATA_ITEM_TM0R6S8 + TEST_DATA_ITEM_TSYCGY5 + TEST_DATA_ITEM_TX39352 + TEST_DATA_ITEM_TX3BMY5 + TEST_DATA_ITEM_TZ3YK09 + TEST_DATA_ITEM_TZNCGC5 + TEST_DATA_ITEM_V1KSJL0 + TEST_DATA_ITEM_V1L5YX7 + TEST_DATA_ITEM_VC83VJ8 + TEST_DATA_ITEM_VC9V9M0 + TEST_DATA_ITEM_VCZX9T3 + TEST_DATA_ITEM_VCZXR77 + TEST_DATA_ITEM_VDHL6L9 + TEST_DATA_ITEM_VFJRRL1 + TEST_DATA_ITEM_VGC9VR0 + TEST_DATA_ITEM_VHK1X95 + TEST_DATA_ITEM_VJHW0T6 + TEST_DATA_ITEM_VM3JV01 + TEST_DATA_ITEM_VPNFSD1 + TEST_DATA_ITEM_VQPPCY6 + TEST_DATA_ITEM_VR8SD78 + TEST_DATA_ITEM_VR8WDZ2 + TEST_DATA_ITEM_VSYYRR0 + TEST_DATA_ITEM_VT0SRW4 + TEST_DATA_ITEM_VVMW173 + TEST_DATA_ITEM_VVN62S5 + TEST_DATA_ITEM_VY1MTS3 + TEST_DATA_ITEM_W0JYML8 + TEST_DATA_ITEM_W0L8BB7 + TEST_DATA_ITEM_W0LMKN8 + TEST_DATA_ITEM_W0LPR69 + TEST_DATA_ITEM_W1PSL14 + TEST_DATA_ITEM_W28C167 + TEST_DATA_ITEM_W4MKVS5 + TEST_DATA_ITEM_W5FQPS6 + TEST_DATA_ITEM_W5FR5R0 + TEST_DATA_ITEM_W5FRCX8 + TEST_DATA_ITEM_W5FT8V7 + TEST_DATA_ITEM_W7G3Z50 + TEST_DATA_ITEM_W7W5NY8 + TEST_DATA_ITEM_W9LF131 + TEST_DATA_ITEM_W9M2XH8 + TEST_DATA_ITEM_W9MJ4N7 + TEST_DATA_ITEM_W9MJLJ4 + TEST_DATA_ITEM_WBPFZD3 + TEST_DATA_ITEM_WC7JCM9 + TEST_DATA_ITEM_WCNCWN6 + TEST_DATA_ITEM_WF4DL58 + TEST_DATA_ITEM_WFDDQN7 + TEST_DATA_ITEM_WFDQ5W9 + TEST_DATA_ITEM_WGHHQ80 + TEST_DATA_ITEM_WGHZBR2 + TEST_DATA_ITEM_WH4KZQ4 + TEST_DATA_ITEM_WH4MGH4 + TEST_DATA_ITEM_WHNLTJ3 + TEST_DATA_ITEM_WKGSRH3 + TEST_DATA_ITEM_WLP74S4 + TEST_DATA_ITEM_WMM5876 + TEST_DATA_ITEM_WPR8S02 + TEST_DATA_ITEM_WS568W8 + TEST_DATA_ITEM_WS5FLJ4 + TEST_DATA_ITEM_WVTFMN1 + TEST_DATA_ITEM_WXWG1T1 + TEST_DATA_ITEM_WXXM0P8 + TEST_DATA_ITEM_WXXTZW6 + TEST_DATA_ITEM_WXXVFD8 + TEST_DATA_ITEM_WYFYGR6 + TEST_DATA_ITEM_WYYG4G4 + TEST_DATA_ITEM_X26Y450 + TEST_DATA_ITEM_X28TQX2 + TEST_DATA_ITEM_X2VGW71 + TEST_DATA_ITEM_X4SX399 + TEST_DATA_ITEM_X5DC287 + TEST_DATA_ITEM_X5DCW09 + TEST_DATA_ITEM_X5FWXL7 + TEST_DATA_ITEM_X5FXHR5 + TEST_DATA_ITEM_X71SNK4 + TEST_DATA_ITEM_X7L19P5 + TEST_DATA_ITEM_X8SGGD0 + TEST_DATA_ITEM_X8SGHP5 + TEST_DATA_ITEM_XDJGZD2 + TEST_DATA_ITEM_XGCH665 + TEST_DATA_ITEM_XH5MMJ4 + TEST_DATA_ITEM_XH5SJJ5 + TEST_DATA_ITEM_XH5T078 + TEST_DATA_ITEM_XK36564 + TEST_DATA_ITEM_XK3WRP6 + TEST_DATA_ITEM_XMLRMW3 + TEST_DATA_ITEM_XMLSQ71 + TEST_DATA_ITEM_XMLV4R4 + TEST_DATA_ITEM_XMLW5H1 + TEST_DATA_ITEM_XP82TZ8 + TEST_DATA_ITEM_XP8Q7F2 + TEST_DATA_ITEM_XRJLX46 + TEST_DATA_ITEM_XRRQ5Y1 + TEST_DATA_ITEM_XRTFPT1 + TEST_DATA_ITEM_XRTJ094 + TEST_DATA_ITEM_XRTJ7G1 + TEST_DATA_ITEM_XS6R1W6 + TEST_DATA_ITEM_XS6SDV0 + TEST_DATA_ITEM_XT27V69 + TEST_DATA_ITEM_XT2CWY0 + TEST_DATA_ITEM_XTHQ679 + TEST_DATA_ITEM_XVBZ6C3 + TEST_DATA_ITEM_XWYLJZ4 + TEST_DATA_ITEM_XX9HGP0 + TEST_DATA_ITEM_XZQN201 + TEST_DATA_ITEM_Y04SBD7 + TEST_DATA_ITEM_Y1DFTW0 + TEST_DATA_ITEM_Y1TJTP9 + TEST_DATA_ITEM_Y29RBS9 + TEST_DATA_ITEM_Y2PX830 + TEST_DATA_ITEM_Y3M19V8 + TEST_DATA_ITEM_Y48N588 + TEST_DATA_ITEM_Y62WBN9 + TEST_DATA_ITEM_Y6G6NT5 + TEST_DATA_ITEM_Y73FBY3 + TEST_DATA_ITEM_Y73WNL2 + TEST_DATA_ITEM_Y7BR4D1 + TEST_DATA_ITEM_Y9DFXF2 + TEST_DATA_ITEM_Y9DLP26 + TEST_DATA_ITEM_Y9DNMM9 + TEST_DATA_ITEM_YBBLNL1 + TEST_DATA_ITEM_YD8K3L1 + TEST_DATA_ITEM_YD93FW9 + TEST_DATA_ITEM_YDGX7G6 + TEST_DATA_ITEM_YFGTVH5 + TEST_DATA_ITEM_YHYK1P0 + TEST_DATA_ITEM_YJKWWL1 + TEST_DATA_ITEM_YM42N63 + TEST_DATA_ITEM_YM46D25 + TEST_DATA_ITEM_YMWFTN5 + TEST_DATA_ITEM_YN8LKB2 + TEST_DATA_ITEM_YN8MVQ1 + TEST_DATA_ITEM_YNK8QZ1 + TEST_DATA_ITEM_YNK8VV4 + TEST_DATA_ITEM_YPSX189 + TEST_DATA_ITEM_YQ7XC99 + TEST_DATA_ITEM_YQ821Y7 + TEST_DATA_ITEM_YRHKZX2 + TEST_DATA_ITEM_YRJDQG5 + TEST_DATA_ITEM_YRJXFY7 + TEST_DATA_ITEM_YT4YFD0 + TEST_DATA_ITEM_YTS8ZC3 + TEST_DATA_ITEM_YTSD463 + TEST_DATA_ITEM_YTSSL78 + TEST_DATA_ITEM_YV1J506 + TEST_DATA_ITEM_YVWDX76 + TEST_DATA_ITEM_YYC3F04 + TEST_DATA_ITEM_YYCW069 + TEST_DATA_ITEM_YZ6J7F1 + TEST_DATA_ITEM_YZF4N35 + TEST_DATA_ITEM_YZF9M32 + TEST_DATA_ITEM_Z0FDB09 + TEST_DATA_ITEM_Z0GR9T6 + TEST_DATA_ITEM_Z0HPBH4 + TEST_DATA_ITEM_Z14GX93 + TEST_DATA_ITEM_Z1JKRH9 + TEST_DATA_ITEM_Z1JTC65 + TEST_DATA_ITEM_Z1JYD85 + TEST_DATA_ITEM_Z1SKF12 + TEST_DATA_ITEM_Z3VW193 + TEST_DATA_ITEM_Z3XTC80 + TEST_DATA_ITEM_Z40QKV0 + TEST_DATA_ITEM_Z5JKRL0 + TEST_DATA_ITEM_Z61T886 + TEST_DATA_ITEM_Z6F6PC7 + TEST_DATA_ITEM_Z6F7VR7 + TEST_DATA_ITEM_Z73NSG4 + TEST_DATA_ITEM_Z853026 + TEST_DATA_ITEM_Z8MRQ26 + TEST_DATA_ITEM_Z917FG2 + TEST_DATA_ITEM_Z9FKSV1 + TEST_DATA_ITEM_Z9FPZG7 + TEST_DATA_ITEM_ZF0FCL9 + TEST_DATA_ITEM_ZF3LDK5 + TEST_DATA_ITEM_ZGD6893 + TEST_DATA_ITEM_ZGF70F7 + TEST_DATA_ITEM_ZHDHTJ3 + TEST_DATA_ITEM_ZHSBLN8 + TEST_DATA_ITEM_ZJ0DR89 + TEST_DATA_ITEM_ZLHY3P0 + TEST_DATA_ITEM_ZLK1486 + TEST_DATA_ITEM_ZNLTB86 + TEST_DATA_ITEM_ZNR97S2 + TEST_DATA_ITEM_ZNSSHK5 + TEST_DATA_ITEM_ZR8BLY9 + TEST_DATA_ITEM_ZS44KX8 + TEST_DATA_ITEM_ZSDRY81 + TEST_DATA_ITEM_ZV0HS43 + TEST_DATA_ITEM_ZVN6KK4 + TEST_DATA_ITEM_ZVWZY77 + TEST_DATA_ITEM_ZVXJTK9 + TEST_DATA_ITEM_ZXQB4R8 + TEST_DATA_ITEM_ZXYXM97 + TEST_DATA_ITEM_100SH258 + TEST_DATA_ITEM_105XBR17 + TEST_DATA_ITEM_10FNXJ58 + TEST_DATA_ITEM_10JW9H28 + TEST_DATA_ITEM_10MVVTT6 + TEST_DATA_ITEM_10MVZF58 + TEST_DATA_ITEM_10WX4G55 + TEST_DATA_ITEM_10WX7YR9 + TEST_DATA_ITEM_1103B266 + TEST_DATA_ITEM_112Z58F7 + TEST_DATA_ITEM_1138D402 + TEST_DATA_ITEM_1163K2N1 + TEST_DATA_ITEM_116YM983 + TEST_DATA_ITEM_118JXC21 + TEST_DATA_ITEM_118LMWQ8 + TEST_DATA_ITEM_118Y8193 + TEST_DATA_ITEM_11BPJW25 + TEST_DATA_ITEM_11C0FQ08 + TEST_DATA_ITEM_11C0PBJ0 + TEST_DATA_ITEM_11C8DJL6 + TEST_DATA_ITEM_11FS2HK6 + TEST_DATA_ITEM_11J3GCN0 + TEST_DATA_ITEM_11K4VZM9 + TEST_DATA_ITEM_11M1TX41 + TEST_DATA_ITEM_11MGV9T1 + TEST_DATA_ITEM_11N542L7 + TEST_DATA_ITEM_11PCQVV1 + TEST_DATA_ITEM_11PFKDM0 + TEST_DATA_ITEM_11PSNRN7 + TEST_DATA_ITEM_11PTS335 + TEST_DATA_ITEM_11RH8527 + TEST_DATA_ITEM_11RK3G67 + TEST_DATA_ITEM_11RT3MR2 + TEST_DATA_ITEM_11RWTSM9 + TEST_DATA_ITEM_11XQL4N5 + TEST_DATA_ITEM_120TBYK1 + TEST_DATA_ITEM_120W16V4 + TEST_DATA_ITEM_12232039 + TEST_DATA_ITEM_122LQP47 + TEST_DATA_ITEM_122MC5Q0 + TEST_DATA_ITEM_123Q6TT2 + TEST_DATA_ITEM_1282Y9T3 + TEST_DATA_ITEM_1282ZJL8 + TEST_DATA_ITEM_129CWDS4 + TEST_DATA_ITEM_12BTZFT5 + TEST_DATA_ITEM_12BV9QP9 + TEST_DATA_ITEM_12C73NW6 + TEST_DATA_ITEM_12C7JPP2 + TEST_DATA_ITEM_12F6LZY6 + TEST_DATA_ITEM_12FCPQP9 + TEST_DATA_ITEM_12MBDC31 + TEST_DATA_ITEM_12MBFPV9 + TEST_DATA_ITEM_12PPWX76 + TEST_DATA_ITEM_12QFVS97 + TEST_DATA_ITEM_12QSDL49 + TEST_DATA_ITEM_12QVB1C9 + TEST_DATA_ITEM_12QX0K80 + TEST_DATA_ITEM_12R3SVM5 + TEST_DATA_ITEM_12THJBR1 + TEST_DATA_ITEM_12TKHH48 + TEST_DATA_ITEM_12V1MCV2 + TEST_DATA_ITEM_12V27BM9 + TEST_DATA_ITEM_12VBH621 + TEST_DATA_ITEM_12W99CF7 + TEST_DATA_ITEM_12WFRGG3 + TEST_DATA_ITEM_12Y71RB8 + TEST_DATA_ITEM_12YL4F48 + TEST_DATA_ITEM_134WCD50 + TEST_DATA_ITEM_135B2107 + TEST_DATA_ITEM_135W7XZ8 + TEST_DATA_ITEM_136KJJM5 + TEST_DATA_ITEM_13978WW3 + TEST_DATA_ITEM_139RY7J0 + TEST_DATA_ITEM_13CYT627 + TEST_DATA_ITEM_13GDWQH5 + TEST_DATA_ITEM_13KKBWZ4 + TEST_DATA_ITEM_13P11C13 + TEST_DATA_ITEM_13P1D8K8 + TEST_DATA_ITEM_13PL2VH9 + TEST_DATA_ITEM_13PRXVT9 + TEST_DATA_ITEM_13THLBR6 + TEST_DATA_ITEM_13ZGNYL5 + TEST_DATA_ITEM_145KKRP1 + TEST_DATA_ITEM_146YWG26 + TEST_DATA_ITEM_149QCWP2 + TEST_DATA_ITEM_14GJ8M40 + TEST_DATA_ITEM_14GJSJM5 + TEST_DATA_ITEM_14J211L8 + TEST_DATA_ITEM_14KFQZV4 + TEST_DATA_ITEM_14QJ4014 + TEST_DATA_ITEM_157C5S06 + TEST_DATA_ITEM_15H3RLY9 + TEST_DATA_ITEM_15LHYNV4 + TEST_DATA_ITEM_160DYBM0 + TEST_DATA_ITEM_16HRS1C1 + TEST_DATA_ITEM_1779WNW5 + TEST_DATA_ITEM_17BXPX26 + TEST_DATA_ITEM_17J18SJ0 + TEST_DATA_ITEM_17J1X165 + TEST_DATA_ITEM_17WSF072 + TEST_DATA_ITEM_17XFZWY7 + TEST_DATA_ITEM_19Y3V5C6 + TEST_DATA_ITEM_1B79XPY8 + + + + DATA.1 + + + DATA.2 + + + + + + + + TEST_DATA_ITEM_1K5SYM4 + TEST_DATA_ITEM_1K612R6 + TEST_DATA_ITEM_1K613S3 + TEST_DATA_ITEM_1K61W18 + TEST_DATA_ITEM_1K82SR4 + TEST_DATA_ITEM_1KS9432 + TEST_DATA_ITEM_1KWWRB7 + TEST_DATA_ITEM_1LH7712 + TEST_DATA_ITEM_1LWHLC5 + TEST_DATA_ITEM_1M6CZQ0 + TEST_DATA_ITEM_1MB89S0 + TEST_DATA_ITEM_1MC8Y67 + TEST_DATA_ITEM_1MKZGX8 + TEST_DATA_ITEM_1MMP7G4 + TEST_DATA_ITEM_1MP3370 + TEST_DATA_ITEM_1NBL4W0 + TEST_DATA_ITEM_1NDW105 + TEST_DATA_ITEM_1NFHCM9 + TEST_DATA_ITEM_1NY19C3 + TEST_DATA_ITEM_1P2KS92 + TEST_DATA_ITEM_1P6MYS7 + TEST_DATA_ITEM_1P71798 + TEST_DATA_ITEM_1PG5WP1 + TEST_DATA_ITEM_1PKMHX7 + TEST_DATA_ITEM_1PM3QY3 + TEST_DATA_ITEM_1Q7HP54 + TEST_DATA_ITEM_1QB7QN4 + TEST_DATA_ITEM_1QD41J3 + TEST_DATA_ITEM_1QJ1RF0 + TEST_DATA_ITEM_1QVVHP1 + TEST_DATA_ITEM_1QY89L3 + TEST_DATA_ITEM_1QYK8Q6 + TEST_DATA_ITEM_1QYNQV7 + TEST_DATA_ITEM_1R1GCP4 + TEST_DATA_ITEM_1R72SJ8 + TEST_DATA_ITEM_1RWPDP4 + TEST_DATA_ITEM_1S58GV7 + TEST_DATA_ITEM_1V6BP01 + TEST_DATA_ITEM_1VDB5Z2 + TEST_DATA_ITEM_1WMKH49 + TEST_DATA_ITEM_1WTB7Z2 + TEST_DATA_ITEM_1WWJKR7 + TEST_DATA_ITEM_1WWJTH9 + TEST_DATA_ITEM_1XJRKX9 + TEST_DATA_ITEM_1Y04TM7 + TEST_DATA_ITEM_1Y2XRZ1 + TEST_DATA_ITEM_1YH8PP1 + TEST_DATA_ITEM_1YKDN46 + TEST_DATA_ITEM_1YR4185 + TEST_DATA_ITEM_1YV1RR1 + TEST_DATA_ITEM_1Z5YJJ8 + TEST_DATA_ITEM_1ZM5R23 + TEST_DATA_ITEM_1ZR5V14 + TEST_DATA_ITEM_1ZZPJB0 + TEST_DATA_ITEM_2030MN4 + TEST_DATA_ITEM_209SZH9 + TEST_DATA_ITEM_212PVF7 + TEST_DATA_ITEM_21C31Z3 + TEST_DATA_ITEM_21H6XV3 + TEST_DATA_ITEM_21J73G1 + TEST_DATA_ITEM_21PB8C5 + TEST_DATA_ITEM_21PH401 + TEST_DATA_ITEM_21RGKR3 + TEST_DATA_ITEM_225ZCZ8 + TEST_DATA_ITEM_226DF92 + TEST_DATA_ITEM_2293PH6 + TEST_DATA_ITEM_22FDRM1 + TEST_DATA_ITEM_22MMM47 + TEST_DATA_ITEM_22PRZQ2 + TEST_DATA_ITEM_23XX752 + TEST_DATA_ITEM_2537T66 + TEST_DATA_ITEM_2583CR3 + TEST_DATA_ITEM_259NGW4 + TEST_DATA_ITEM_259SBC2 + TEST_DATA_ITEM_25DTRD6 + TEST_DATA_ITEM_25X1684 + TEST_DATA_ITEM_25XVR58 + TEST_DATA_ITEM_25Y4RX5 + TEST_DATA_ITEM_265J747 + TEST_DATA_ITEM_265J890 + TEST_DATA_ITEM_265K264 + TEST_DATA_ITEM_265T2Y4 + TEST_DATA_ITEM_265XT61 + TEST_DATA_ITEM_265Y0D5 + TEST_DATA_ITEM_265Y385 + TEST_DATA_ITEM_265Y3R4 + TEST_DATA_ITEM_265Y492 + TEST_DATA_ITEM_265Y4K9 + TEST_DATA_ITEM_27F2B70 + TEST_DATA_ITEM_27NTVR8 + TEST_DATA_ITEM_27Z6LY0 + TEST_DATA_ITEM_2832GJ2 + TEST_DATA_ITEM_283NM33 + TEST_DATA_ITEM_286RYD5 + TEST_DATA_ITEM_29SNNW3 + TEST_DATA_ITEM_29ZX804 + TEST_DATA_ITEM_2B04MD5 + TEST_DATA_ITEM_2BC7W26 + TEST_DATA_ITEM_2CHBLY8 + TEST_DATA_ITEM_2CN8XH2 + TEST_DATA_ITEM_2CV4Q65 + TEST_DATA_ITEM_2CV5W52 + TEST_DATA_ITEM_2G8Q1G2 + TEST_DATA_ITEM_2GBMZF5 + TEST_DATA_ITEM_2GKBN64 + TEST_DATA_ITEM_2H242T2 + TEST_DATA_ITEM_2N863Q1 + TEST_DATA_ITEM_2NC3XW2 + TEST_DATA_ITEM_2NCK5C6 + TEST_DATA_ITEM_2PY4KR5 + TEST_DATA_ITEM_2Q7DD08 + TEST_DATA_ITEM_2R1CW18 + TEST_DATA_ITEM_2R1JYX2 + TEST_DATA_ITEM_2TWLBK3 + TEST_DATA_ITEM_2V08ZP7 + TEST_DATA_ITEM_2V87RM2 + TEST_DATA_ITEM_2VNPMC1 + TEST_DATA_ITEM_2VZ68D5 + TEST_DATA_ITEM_2WLDMR1 + TEST_DATA_ITEM_2WN7DS3 + TEST_DATA_ITEM_2Y9YZ03 + TEST_DATA_ITEM_2YSBH33 + TEST_DATA_ITEM_2ZCK2F7 + TEST_DATA_ITEM_2ZHMDC3 + TEST_DATA_ITEM_2ZYMKP6 + TEST_DATA_ITEM_328Q6M7 + TEST_DATA_ITEM_331G2L2 + TEST_DATA_ITEM_3338H16 + TEST_DATA_ITEM_34VYL97 + TEST_DATA_ITEM_35LY8V2 + TEST_DATA_ITEM_38J7Q35 + TEST_DATA_ITEM_39320K2 + TEST_DATA_ITEM_393H570 + TEST_DATA_ITEM_3981NR5 + TEST_DATA_ITEM_3BC3JN4 + TEST_DATA_ITEM_3BW05J8 + TEST_DATA_ITEM_3CSG3N9 + TEST_DATA_ITEM_3CVJNM6 + TEST_DATA_ITEM_3CVMLJ0 + TEST_DATA_ITEM_3D4V915 + TEST_DATA_ITEM_3DNHV47 + TEST_DATA_ITEM_3FD3647 + TEST_DATA_ITEM_3GMH2C1 + TEST_DATA_ITEM_3LF0QF2 + TEST_DATA_ITEM_3LFWLS4 + TEST_DATA_ITEM_3M6RTM8 + TEST_DATA_ITEM_3MKSKY2 + TEST_DATA_ITEM_3MSMCM2 + TEST_DATA_ITEM_3NJHZK8 + TEST_DATA_ITEM_3NXJMR7 + TEST_DATA_ITEM_3PHHZR3 + TEST_DATA_ITEM_3PS7JG8 + TEST_DATA_ITEM_3Q5HK19 + TEST_DATA_ITEM_3RY8199 + TEST_DATA_ITEM_3S43W19 + TEST_DATA_ITEM_3T1GLD1 + TEST_DATA_ITEM_3TCRP91 + TEST_DATA_ITEM_42V6B90 + TEST_DATA_ITEM_42V8KB5 + TEST_DATA_ITEM_4332MG3 + TEST_DATA_ITEM_4336Q52 + TEST_DATA_ITEM_43BYP46 + TEST_DATA_ITEM_443M531 + TEST_DATA_ITEM_44BWGM7 + TEST_DATA_ITEM_44JTND9 + TEST_DATA_ITEM_44K5CS0 + TEST_DATA_ITEM_46L1KK0 + TEST_DATA_ITEM_46M36P2 + TEST_DATA_ITEM_47C19F0 + TEST_DATA_ITEM_4CYX1J4 + TEST_DATA_ITEM_4DRTM89 + TEST_DATA_ITEM_4FPDKG3 + TEST_DATA_ITEM_4HK6T53 + TEST_DATA_ITEM_4HQHKF6 + TEST_DATA_ITEM_4HTLQ62 + TEST_DATA_ITEM_4HTMCD4 + TEST_DATA_ITEM_4HTNF91 + TEST_DATA_ITEM_4K27NX0 + TEST_DATA_ITEM_4KB3S09 + TEST_DATA_ITEM_4KDF0Z5 + TEST_DATA_ITEM_4LT48C9 + TEST_DATA_ITEM_4M8M6F5 + TEST_DATA_ITEM_4M9ZHB9 + TEST_DATA_ITEM_4MB8147 + TEST_DATA_ITEM_4MF59G1 + TEST_DATA_ITEM_4MN1R23 + TEST_DATA_ITEM_4NLQDL9 + TEST_DATA_ITEM_4P33P44 + TEST_DATA_ITEM_4PKT0F1 + TEST_DATA_ITEM_4PW84C3 + TEST_DATA_ITEM_4Q00KR8 + TEST_DATA_ITEM_4S69ZV6 + TEST_DATA_ITEM_4SCBZS4 + TEST_DATA_ITEM_4SK5VK0 + TEST_DATA_ITEM_4SLD8Y5 + TEST_DATA_ITEM_4TYNXJ7 + TEST_DATA_ITEM_4WQKBX5 + TEST_DATA_ITEM_4X2QSY3 + TEST_DATA_ITEM_4Z1PVR5 + TEST_DATA_ITEM_53Z92J3 + TEST_DATA_ITEM_5497GX5 + TEST_DATA_ITEM_5544NR0 + TEST_DATA_ITEM_55FJVD6 + TEST_DATA_ITEM_56655N6 + TEST_DATA_ITEM_56JW5F7 + TEST_DATA_ITEM_57SDZW2 + TEST_DATA_ITEM_58KMGM1 + TEST_DATA_ITEM_5914G24 + TEST_DATA_ITEM_59FN6V2 + TEST_DATA_ITEM_5BLN156 + TEST_DATA_ITEM_5BTNF94 + TEST_DATA_ITEM_5C9FQB9 + TEST_DATA_ITEM_5CHLM23 + TEST_DATA_ITEM_5CHSS23 + TEST_DATA_ITEM_5CX4HJ0 + TEST_DATA_ITEM_5D1Y4M1 + TEST_DATA_ITEM_5D7PD75 + TEST_DATA_ITEM_5D7QCC0 + TEST_DATA_ITEM_5DJFD34 + TEST_DATA_ITEM_5DWN8J0 + TEST_DATA_ITEM_5H82FL3 + TEST_DATA_ITEM_5HTCSZ3 + TEST_DATA_ITEM_5KC3JN9 + TEST_DATA_ITEM_5KC4Z34 + TEST_DATA_ITEM_5L86FY0 + TEST_DATA_ITEM_5M4HFS3 + TEST_DATA_ITEM_5MX5GQ2 + TEST_DATA_ITEM_5NTTSH8 + TEST_DATA_ITEM_5P33368 + TEST_DATA_ITEM_5PG80F0 + TEST_DATA_ITEM_5PXTR52 + TEST_DATA_ITEM_5Q22H32 + TEST_DATA_ITEM_5Q29TT1 + TEST_DATA_ITEM_5Q3MQ74 + TEST_DATA_ITEM_5RVGYZ0 + TEST_DATA_ITEM_5S5X7T3 + TEST_DATA_ITEM_5T10SB0 + TEST_DATA_ITEM_5T5RWP8 + TEST_DATA_ITEM_5TJKDX8 + TEST_DATA_ITEM_5TNRYD8 + TEST_DATA_ITEM_5WQVSV5 + TEST_DATA_ITEM_5WX31J4 + TEST_DATA_ITEM_5WX3GZ3 + TEST_DATA_ITEM_5XKG704 + TEST_DATA_ITEM_5XNT5S8 + TEST_DATA_ITEM_5XR47G5 + TEST_DATA_ITEM_5XVML57 + TEST_DATA_ITEM_5XVXMD4 + TEST_DATA_ITEM_5XZHQZ0 + TEST_DATA_ITEM_5ZP9905 + TEST_DATA_ITEM_5ZRM1T2 + TEST_DATA_ITEM_5ZV19Y3 + TEST_DATA_ITEM_5ZVD8W4 + TEST_DATA_ITEM_5ZVK9M5 + TEST_DATA_ITEM_60B3LG4 + TEST_DATA_ITEM_60CNMF9 + TEST_DATA_ITEM_629NGG6 + TEST_DATA_ITEM_63GCHD2 + TEST_DATA_ITEM_63N8V50 + TEST_DATA_ITEM_6473QV1 + TEST_DATA_ITEM_64N0ZY6 + TEST_DATA_ITEM_64N2T69 + TEST_DATA_ITEM_658F3L7 + TEST_DATA_ITEM_65B28F0 + TEST_DATA_ITEM_66FZJL3 + TEST_DATA_ITEM_67QVJ32 + TEST_DATA_ITEM_6BFYQM1 + TEST_DATA_ITEM_6BZHZ28 + TEST_DATA_ITEM_6D97TY4 + TEST_DATA_ITEM_6DZTHY8 + TEST_DATA_ITEM_6F8QZF0 + TEST_DATA_ITEM_6FC9ZZ8 + TEST_DATA_ITEM_6GNRZ10 + TEST_DATA_ITEM_6HFPX68 + TEST_DATA_ITEM_6JS5776 + TEST_DATA_ITEM_6KDCG60 + TEST_DATA_ITEM_6KZRJY8 + TEST_DATA_ITEM_6MC4LM0 + TEST_DATA_ITEM_6MDLXX1 + TEST_DATA_ITEM_6N7S1J2 + TEST_DATA_ITEM_6NM92T7 + TEST_DATA_ITEM_6NSMRN0 + TEST_DATA_ITEM_6Q4JR45 + TEST_DATA_ITEM_6Q52QT5 + TEST_DATA_ITEM_6R5SRB3 + TEST_DATA_ITEM_6RB1XG1 + TEST_DATA_ITEM_6S59GX3 + TEST_DATA_ITEM_6SCVC71 + TEST_DATA_ITEM_6XWWBK7 + TEST_DATA_ITEM_6YWVSN7 + TEST_DATA_ITEM_6Z564Q3 + TEST_DATA_ITEM_70J7ZY0 + TEST_DATA_ITEM_71HZ630 + TEST_DATA_ITEM_7318RQ2 + TEST_DATA_ITEM_73F9RP1 + TEST_DATA_ITEM_74JPBL3 + TEST_DATA_ITEM_74Q3NJ8 + TEST_DATA_ITEM_74VSJH1 + TEST_DATA_ITEM_77Q6Q50 + TEST_DATA_ITEM_77VNXK8 + TEST_DATA_ITEM_77VS187 + TEST_DATA_ITEM_78W3N65 + TEST_DATA_ITEM_78Y1D74 + TEST_DATA_ITEM_79T1K69 + TEST_DATA_ITEM_7BVZ8G0 + TEST_DATA_ITEM_7DHG843 + TEST_DATA_ITEM_7DHXQL5 + TEST_DATA_ITEM_7FG09F6 + TEST_DATA_ITEM_7FH8JR1 + TEST_DATA_ITEM_7H0TDD1 + TEST_DATA_ITEM_7HRJX03 + TEST_DATA_ITEM_7HWGTZ6 + TEST_DATA_ITEM_7J2XLR7 + TEST_DATA_ITEM_7KC7NM3 + TEST_DATA_ITEM_7KGRPK9 + TEST_DATA_ITEM_7NJLS20 + TEST_DATA_ITEM_7NLG3D4 + TEST_DATA_ITEM_7NM37N6 + TEST_DATA_ITEM_7NSWGH8 + TEST_DATA_ITEM_7QZBG64 + TEST_DATA_ITEM_7SV91S3 + TEST_DATA_ITEM_7T9DYL1 + TEST_DATA_ITEM_7TJF1M8 + TEST_DATA_ITEM_7W9BP38 + TEST_DATA_ITEM_7WX14V2 + TEST_DATA_ITEM_7Z9TCY1 + TEST_DATA_ITEM_81VHT64 + TEST_DATA_ITEM_829DZ71 + TEST_DATA_ITEM_8417VM5 + TEST_DATA_ITEM_87F73K5 + TEST_DATA_ITEM_8974324 + TEST_DATA_ITEM_8B2CXR0 + TEST_DATA_ITEM_8B7FKG6 + TEST_DATA_ITEM_8CS2P27 + TEST_DATA_ITEM_8G28L69 + TEST_DATA_ITEM_8G5ZX75 + TEST_DATA_ITEM_8HNK0Q2 + TEST_DATA_ITEM_8HNZ816 + TEST_DATA_ITEM_8N1BXX3 + TEST_DATA_ITEM_8N1HWF9 + TEST_DATA_ITEM_8N29887 + TEST_DATA_ITEM_8NMBXM9 + TEST_DATA_ITEM_8NZ8PY8 + TEST_DATA_ITEM_8P7F832 + TEST_DATA_ITEM_8QPZ964 + TEST_DATA_ITEM_98VP8P1 + TEST_DATA_ITEM_99X6840 + TEST_DATA_ITEM_9DFHW85 + TEST_DATA_ITEM_9DMF643 + TEST_DATA_ITEM_9DVZS89 + TEST_DATA_ITEM_9F3H5C4 + TEST_DATA_ITEM_9H06RN0 + TEST_DATA_ITEM_9H33P42 + TEST_DATA_ITEM_9HYYCC0 + TEST_DATA_ITEM_9J8K5H0 + TEST_DATA_ITEM_9KD84C9 + TEST_DATA_ITEM_9KG16X1 + TEST_DATA_ITEM_9L0WFF1 + TEST_DATA_ITEM_9LR89M5 + TEST_DATA_ITEM_9LVKPB8 + TEST_DATA_ITEM_9NMSGF4 + TEST_DATA_ITEM_9PHHNW1 + TEST_DATA_ITEM_9PN0BZ9 + TEST_DATA_ITEM_9Q03659 + TEST_DATA_ITEM_9R0CT30 + TEST_DATA_ITEM_9R48CV3 + TEST_DATA_ITEM_9S1XNF4 + TEST_DATA_ITEM_9SYBY62 + TEST_DATA_ITEM_9T0TKT1 + TEST_DATA_ITEM_9T223N0 + TEST_DATA_ITEM_9W8YNM5 + TEST_DATA_ITEM_9XW2W00 + TEST_DATA_ITEM_B3T3GZ1 + TEST_DATA_ITEM_B5ZT2C3 + TEST_DATA_ITEM_B6F29Z5 + TEST_DATA_ITEM_B6G8059 + TEST_DATA_ITEM_B6QMZR0 + TEST_DATA_ITEM_B9N3KX1 + TEST_DATA_ITEM_BC58RL9 + TEST_DATA_ITEM_BCHPRL4 + TEST_DATA_ITEM_BCQJ2N4 + TEST_DATA_ITEM_BDCCTY0 + TEST_DATA_ITEM_BFHD177 + TEST_DATA_ITEM_BFHD6J3 + TEST_DATA_ITEM_BFHDCL7 + TEST_DATA_ITEM_BLVZ1W7 + TEST_DATA_ITEM_BLYKRY8 + TEST_DATA_ITEM_BM5XVM0 + TEST_DATA_ITEM_BN8S8L1 + TEST_DATA_ITEM_BN96557 + TEST_DATA_ITEM_BN965M8 + TEST_DATA_ITEM_BNTV528 + TEST_DATA_ITEM_BR9Q533 + TEST_DATA_ITEM_BS050G1 + TEST_DATA_ITEM_BTJVJN1 + TEST_DATA_ITEM_BV2Y8C8 + TEST_DATA_ITEM_BVR0CS4 + TEST_DATA_ITEM_C0NV622 + TEST_DATA_ITEM_C1BZMM6 + TEST_DATA_ITEM_C4DL550 + TEST_DATA_ITEM_C764D45 + TEST_DATA_ITEM_C77LF85 + TEST_DATA_ITEM_CBYY6J5 + TEST_DATA_ITEM_CCNTJ06 + TEST_DATA_ITEM_CP9PCB0 + TEST_DATA_ITEM_CSDQMB5 + TEST_DATA_ITEM_CSG01S4 + TEST_DATA_ITEM_CTNNT93 + TEST_DATA_ITEM_CTZ4C49 + TEST_DATA_ITEM_CVQYJN7 + TEST_DATA_ITEM_CW0SNV2 + TEST_DATA_ITEM_CW88RB0 + TEST_DATA_ITEM_CWTTPF1 + TEST_DATA_ITEM_CZNLQX7 + TEST_DATA_ITEM_D0Y8150 + TEST_DATA_ITEM_D0YK7S8 + TEST_DATA_ITEM_D2Z7X38 + TEST_DATA_ITEM_D30HGG6 + TEST_DATA_ITEM_D36Q0Y8 + TEST_DATA_ITEM_D5L3SQ6 + TEST_DATA_ITEM_D854W79 + TEST_DATA_ITEM_D88BWN7 + TEST_DATA_ITEM_D8JC855 + TEST_DATA_ITEM_NN874Q2 + TEST_DATA_ITEM_NN9HMZ9 + TEST_DATA_ITEM_NNW8CC5 + TEST_DATA_ITEM_NSBBST9 + TEST_DATA_ITEM_NSGPHL5 + TEST_DATA_ITEM_NTM77W1 + TEST_DATA_ITEM_NTMFCY9 + TEST_DATA_ITEM_NWCBRS5 + TEST_DATA_ITEM_NWR7XX8 + TEST_DATA_ITEM_NX1ZL23 + TEST_DATA_ITEM_P0YVWW9 + TEST_DATA_ITEM_P2HLM57 + TEST_DATA_ITEM_P33TC79 + TEST_DATA_ITEM_P4JBS77 + TEST_DATA_ITEM_P4R2RX2 + TEST_DATA_ITEM_P4V6488 + TEST_DATA_ITEM_P5B2Q95 + TEST_DATA_ITEM_P80SD42 + TEST_DATA_ITEM_P99J4F0 + TEST_DATA_ITEM_P9KDGX4 + TEST_DATA_ITEM_P9KHS64 + TEST_DATA_ITEM_PBD6TY6 + TEST_DATA_ITEM_PDYP436 + TEST_DATA_ITEM_PF4SSD3 + TEST_DATA_ITEM_PFB4QJ1 + TEST_DATA_ITEM_PGH1GH4 + TEST_DATA_ITEM_PGK4SL3 + TEST_DATA_ITEM_PKCBY08 + TEST_DATA_ITEM_PKGL404 + TEST_DATA_ITEM_PLQ0S01 + TEST_DATA_ITEM_PM24Z12 + TEST_DATA_ITEM_PM256H8 + TEST_DATA_ITEM_PM8N596 + TEST_DATA_ITEM_PNFCVP6 + TEST_DATA_ITEM_PNN7288 + TEST_DATA_ITEM_PNR0WT7 + TEST_DATA_ITEM_PPP6FW6 + TEST_DATA_ITEM_PPXGG29 + TEST_DATA_ITEM_PQ4XFB5 + TEST_DATA_ITEM_PQGXHY9 + TEST_DATA_ITEM_PQRZX14 + TEST_DATA_ITEM_PT33734 + TEST_DATA_ITEM_PZV44X8 + TEST_DATA_ITEM_Q011TH6 + TEST_DATA_ITEM_Q3HY9H6 + TEST_DATA_ITEM_Q70LZG6 + TEST_DATA_ITEM_Q72WX48 + TEST_DATA_ITEM_Q8J3R18 + TEST_DATA_ITEM_QDCHBL5 + TEST_DATA_ITEM_QFW4LT3 + TEST_DATA_ITEM_QG40R60 + TEST_DATA_ITEM_QHJM2M9 + TEST_DATA_ITEM_QHK2C82 + TEST_DATA_ITEM_QJ46QB7 + TEST_DATA_ITEM_QJBWHP7 + TEST_DATA_ITEM_QLF1CR8 + TEST_DATA_ITEM_QN8JZX5 + TEST_DATA_ITEM_QQ1XHZ9 + TEST_DATA_ITEM_QQ8C6S9 + TEST_DATA_ITEM_QQ8GRH1 + TEST_DATA_ITEM_QS5PBQ0 + TEST_DATA_ITEM_QV3CB85 + TEST_DATA_ITEM_QVJV494 + TEST_DATA_ITEM_QYLY9P4 + TEST_DATA_ITEM_QYNLWF4 + TEST_DATA_ITEM_R1WNZS5 + TEST_DATA_ITEM_R24W7H0 + TEST_DATA_ITEM_R28WT29 + TEST_DATA_ITEM_R2JZFV0 + TEST_DATA_ITEM_R2K8TC8 + TEST_DATA_ITEM_R4SGV71 + TEST_DATA_ITEM_R7TJRF3 + TEST_DATA_ITEM_RB9B6Y5 + TEST_DATA_ITEM_RBF97K7 + TEST_DATA_ITEM_RBFBJD6 + TEST_DATA_ITEM_RCNNPK3 + TEST_DATA_ITEM_RCQD6F5 + TEST_DATA_ITEM_RD1PF93 + TEST_DATA_ITEM_RDSZ2T0 + TEST_DATA_ITEM_RJXPZR3 + TEST_DATA_ITEM_RKYPJR5 + TEST_DATA_ITEM_RM7XHH8 + TEST_DATA_ITEM_RNXF8T7 + TEST_DATA_ITEM_RP5HKT7 + TEST_DATA_ITEM_RP7FX08 + TEST_DATA_ITEM_RPWYNV0 + TEST_DATA_ITEM_RRGSHJ7 + TEST_DATA_ITEM_RVG4979 + TEST_DATA_ITEM_RZFGN37 + TEST_DATA_ITEM_S03JPK3 + TEST_DATA_ITEM_S1HHY24 + TEST_DATA_ITEM_S1LQXM5 + TEST_DATA_ITEM_S2K1884 + TEST_DATA_ITEM_S2L0213 + TEST_DATA_ITEM_S2L11V1 + TEST_DATA_ITEM_S2L1YW8 + TEST_DATA_ITEM_S42MY19 + TEST_DATA_ITEM_S7P0Q34 + TEST_DATA_ITEM_S7P5H84 + TEST_DATA_ITEM_SBXZ9J5 + TEST_DATA_ITEM_SL59VF9 + TEST_DATA_ITEM_SNC4234 + TEST_DATA_ITEM_SSQFBD5 + TEST_DATA_ITEM_T5JZ7Y8 + TEST_DATA_ITEM_T68GVF0 + TEST_DATA_ITEM_T8B8C35 + TEST_DATA_ITEM_THX1KC6 + TEST_DATA_ITEM_TJHY616 + TEST_DATA_ITEM_TJJ1191 + TEST_DATA_ITEM_TM0R6S8 + TEST_DATA_ITEM_TSYCGY5 + TEST_DATA_ITEM_TX39352 + TEST_DATA_ITEM_TX3BMY5 + TEST_DATA_ITEM_TZ3YK09 + TEST_DATA_ITEM_TZNCGC5 + TEST_DATA_ITEM_V1KSJL0 + TEST_DATA_ITEM_V1L5YX7 + TEST_DATA_ITEM_VC83VJ8 + TEST_DATA_ITEM_VC9V9M0 + TEST_DATA_ITEM_VCZX9T3 + TEST_DATA_ITEM_VCZXR77 + TEST_DATA_ITEM_VDHL6L9 + TEST_DATA_ITEM_VFJRRL1 + TEST_DATA_ITEM_VGC9VR0 + TEST_DATA_ITEM_VHK1X95 + TEST_DATA_ITEM_VJHW0T6 + TEST_DATA_ITEM_VM3JV01 + TEST_DATA_ITEM_VPNFSD1 + TEST_DATA_ITEM_VQPPCY6 + TEST_DATA_ITEM_VR8SD78 + TEST_DATA_ITEM_VR8WDZ2 + TEST_DATA_ITEM_VSYYRR0 + TEST_DATA_ITEM_VT0SRW4 + TEST_DATA_ITEM_VVMW173 + TEST_DATA_ITEM_VVN62S5 + TEST_DATA_ITEM_VY1MTS3 + TEST_DATA_ITEM_W0JYML8 + TEST_DATA_ITEM_W0L8BB7 + TEST_DATA_ITEM_W0LMKN8 + TEST_DATA_ITEM_W0LPR69 + TEST_DATA_ITEM_W1PSL14 + TEST_DATA_ITEM_W28C167 + TEST_DATA_ITEM_W4MKVS5 + TEST_DATA_ITEM_W5FQPS6 + TEST_DATA_ITEM_W5FR5R0 + TEST_DATA_ITEM_W5FRCX8 + TEST_DATA_ITEM_W5FT8V7 + TEST_DATA_ITEM_W7G3Z50 + TEST_DATA_ITEM_W7W5NY8 + TEST_DATA_ITEM_W9LF131 + TEST_DATA_ITEM_W9M2XH8 + TEST_DATA_ITEM_W9MJ4N7 + TEST_DATA_ITEM_W9MJLJ4 + TEST_DATA_ITEM_WBPFZD3 + TEST_DATA_ITEM_WC7JCM9 + TEST_DATA_ITEM_WCNCWN6 + TEST_DATA_ITEM_WF4DL58 + TEST_DATA_ITEM_WFDDQN7 + TEST_DATA_ITEM_WFDQ5W9 + TEST_DATA_ITEM_WGHHQ80 + TEST_DATA_ITEM_WGHZBR2 + TEST_DATA_ITEM_WH4KZQ4 + TEST_DATA_ITEM_WH4MGH4 + TEST_DATA_ITEM_WHNLTJ3 + TEST_DATA_ITEM_WKGSRH3 + TEST_DATA_ITEM_WLP74S4 + TEST_DATA_ITEM_WMM5876 + TEST_DATA_ITEM_WPR8S02 + TEST_DATA_ITEM_WS568W8 + TEST_DATA_ITEM_WS5FLJ4 + TEST_DATA_ITEM_WVTFMN1 + TEST_DATA_ITEM_WXWG1T1 + TEST_DATA_ITEM_WXXM0P8 + TEST_DATA_ITEM_WXXTZW6 + TEST_DATA_ITEM_WXXVFD8 + TEST_DATA_ITEM_WYFYGR6 + TEST_DATA_ITEM_WYYG4G4 + TEST_DATA_ITEM_X26Y450 + TEST_DATA_ITEM_X28TQX2 + TEST_DATA_ITEM_X2VGW71 + TEST_DATA_ITEM_X4SX399 + TEST_DATA_ITEM_X5DC287 + TEST_DATA_ITEM_X5DCW09 + TEST_DATA_ITEM_X5FWXL7 + TEST_DATA_ITEM_X5FXHR5 + TEST_DATA_ITEM_X71SNK4 + TEST_DATA_ITEM_X7L19P5 + TEST_DATA_ITEM_X8SGGD0 + TEST_DATA_ITEM_X8SGHP5 + TEST_DATA_ITEM_XDJGZD2 + TEST_DATA_ITEM_XGCH665 + TEST_DATA_ITEM_XH5MMJ4 + TEST_DATA_ITEM_XH5SJJ5 + TEST_DATA_ITEM_XH5T078 + TEST_DATA_ITEM_XK36564 + TEST_DATA_ITEM_XK3WRP6 + TEST_DATA_ITEM_XMLRMW3 + TEST_DATA_ITEM_XMLSQ71 + TEST_DATA_ITEM_XMLV4R4 + TEST_DATA_ITEM_XMLW5H1 + TEST_DATA_ITEM_XP82TZ8 + TEST_DATA_ITEM_XP8Q7F2 + TEST_DATA_ITEM_XRJLX46 + TEST_DATA_ITEM_XRRQ5Y1 + TEST_DATA_ITEM_XRTFPT1 + TEST_DATA_ITEM_XRTJ094 + TEST_DATA_ITEM_XRTJ7G1 + TEST_DATA_ITEM_XS6R1W6 + TEST_DATA_ITEM_XS6SDV0 + TEST_DATA_ITEM_XT27V69 + TEST_DATA_ITEM_XT2CWY0 + TEST_DATA_ITEM_XTHQ679 + TEST_DATA_ITEM_XVBZ6C3 + TEST_DATA_ITEM_XWYLJZ4 + TEST_DATA_ITEM_XX9HGP0 + TEST_DATA_ITEM_XZQN201 + TEST_DATA_ITEM_Y04SBD7 + TEST_DATA_ITEM_Y1DFTW0 + TEST_DATA_ITEM_Y1TJTP9 + TEST_DATA_ITEM_Y29RBS9 + TEST_DATA_ITEM_Y2PX830 + TEST_DATA_ITEM_Y3M19V8 + TEST_DATA_ITEM_Y48N588 + TEST_DATA_ITEM_Y62WBN9 + TEST_DATA_ITEM_Y6G6NT5 + TEST_DATA_ITEM_Y73FBY3 + TEST_DATA_ITEM_Y73WNL2 + TEST_DATA_ITEM_Y7BR4D1 + TEST_DATA_ITEM_Y9DFXF2 + TEST_DATA_ITEM_Y9DLP26 + TEST_DATA_ITEM_Y9DNMM9 + TEST_DATA_ITEM_YBBLNL1 + TEST_DATA_ITEM_YD8K3L1 + TEST_DATA_ITEM_YD93FW9 + TEST_DATA_ITEM_YDGX7G6 + TEST_DATA_ITEM_YFGTVH5 + TEST_DATA_ITEM_YHYK1P0 + TEST_DATA_ITEM_YJKWWL1 + TEST_DATA_ITEM_YM42N63 + TEST_DATA_ITEM_YM46D25 + TEST_DATA_ITEM_YMWFTN5 + TEST_DATA_ITEM_YN8LKB2 + TEST_DATA_ITEM_YN8MVQ1 + TEST_DATA_ITEM_YNK8QZ1 + TEST_DATA_ITEM_YNK8VV4 + TEST_DATA_ITEM_YPSX189 + TEST_DATA_ITEM_YQ7XC99 + TEST_DATA_ITEM_YQ821Y7 + TEST_DATA_ITEM_YRHKZX2 + TEST_DATA_ITEM_YRJDQG5 + TEST_DATA_ITEM_YRJXFY7 + TEST_DATA_ITEM_YT4YFD0 + TEST_DATA_ITEM_YTS8ZC3 + TEST_DATA_ITEM_YTSD463 + TEST_DATA_ITEM_YTSSL78 + TEST_DATA_ITEM_YV1J506 + TEST_DATA_ITEM_YVWDX76 + TEST_DATA_ITEM_YYC3F04 + TEST_DATA_ITEM_YYCW069 + TEST_DATA_ITEM_YZ6J7F1 + TEST_DATA_ITEM_YZF4N35 + TEST_DATA_ITEM_YZF9M32 + TEST_DATA_ITEM_Z0FDB09 + TEST_DATA_ITEM_Z0GR9T6 + TEST_DATA_ITEM_Z0HPBH4 + TEST_DATA_ITEM_Z14GX93 + TEST_DATA_ITEM_Z1JKRH9 + TEST_DATA_ITEM_Z1JTC65 + TEST_DATA_ITEM_Z1JYD85 + TEST_DATA_ITEM_Z1SKF12 + TEST_DATA_ITEM_Z3VW193 + TEST_DATA_ITEM_Z3XTC80 + TEST_DATA_ITEM_Z40QKV0 + TEST_DATA_ITEM_Z5JKRL0 + TEST_DATA_ITEM_Z61T886 + TEST_DATA_ITEM_Z6F6PC7 + TEST_DATA_ITEM_Z6F7VR7 + TEST_DATA_ITEM_Z73NSG4 + TEST_DATA_ITEM_Z853026 + TEST_DATA_ITEM_Z8MRQ26 + TEST_DATA_ITEM_Z917FG2 + TEST_DATA_ITEM_Z9FKSV1 + TEST_DATA_ITEM_Z9FPZG7 + TEST_DATA_ITEM_ZF0FCL9 + TEST_DATA_ITEM_ZF3LDK5 + TEST_DATA_ITEM_ZGD6893 + TEST_DATA_ITEM_ZGF70F7 + TEST_DATA_ITEM_ZHDHTJ3 + TEST_DATA_ITEM_ZHSBLN8 + TEST_DATA_ITEM_ZJ0DR89 + TEST_DATA_ITEM_ZLHY3P0 + TEST_DATA_ITEM_ZLK1486 + TEST_DATA_ITEM_ZNLTB86 + TEST_DATA_ITEM_ZNR97S2 + TEST_DATA_ITEM_ZNSSHK5 + TEST_DATA_ITEM_ZR8BLY9 + TEST_DATA_ITEM_ZS44KX8 + TEST_DATA_ITEM_ZSDRY81 + TEST_DATA_ITEM_ZV0HS43 + TEST_DATA_ITEM_ZVN6KK4 + TEST_DATA_ITEM_ZVWZY77 + TEST_DATA_ITEM_ZVXJTK9 + TEST_DATA_ITEM_ZXQB4R8 + TEST_DATA_ITEM_ZXYXM97 + TEST_DATA_ITEM_100SH258 + TEST_DATA_ITEM_105XBR17 + TEST_DATA_ITEM_10FNXJ58 + TEST_DATA_ITEM_10JW9H28 + TEST_DATA_ITEM_10MVVTT6 + TEST_DATA_ITEM_10MVZF58 + TEST_DATA_ITEM_10WX4G55 + TEST_DATA_ITEM_10WX7YR9 + TEST_DATA_ITEM_1103B266 + TEST_DATA_ITEM_112Z58F7 + TEST_DATA_ITEM_1138D402 + TEST_DATA_ITEM_1163K2N1 + TEST_DATA_ITEM_116YM983 + TEST_DATA_ITEM_118JXC21 + TEST_DATA_ITEM_118LMWQ8 + TEST_DATA_ITEM_118Y8193 + TEST_DATA_ITEM_11BPJW25 + TEST_DATA_ITEM_11C0FQ08 + TEST_DATA_ITEM_11C0PBJ0 + TEST_DATA_ITEM_11C8DJL6 + TEST_DATA_ITEM_11FS2HK6 + TEST_DATA_ITEM_11J3GCN0 + TEST_DATA_ITEM_11K4VZM9 + TEST_DATA_ITEM_11M1TX41 + TEST_DATA_ITEM_11MGV9T1 + TEST_DATA_ITEM_11N542L7 + TEST_DATA_ITEM_11PCQVV1 + TEST_DATA_ITEM_11PFKDM0 + TEST_DATA_ITEM_11PSNRN7 + TEST_DATA_ITEM_11PTS335 + TEST_DATA_ITEM_11RH8527 + TEST_DATA_ITEM_11RK3G67 + TEST_DATA_ITEM_11RT3MR2 + TEST_DATA_ITEM_11RWTSM9 + TEST_DATA_ITEM_11XQL4N5 + TEST_DATA_ITEM_120TBYK1 + TEST_DATA_ITEM_120W16V4 + TEST_DATA_ITEM_12232039 + TEST_DATA_ITEM_122LQP47 + TEST_DATA_ITEM_122MC5Q0 + TEST_DATA_ITEM_123Q6TT2 + TEST_DATA_ITEM_1282Y9T3 + TEST_DATA_ITEM_1282ZJL8 + TEST_DATA_ITEM_129CWDS4 + TEST_DATA_ITEM_12BTZFT5 + TEST_DATA_ITEM_12BV9QP9 + TEST_DATA_ITEM_12C73NW6 + TEST_DATA_ITEM_12C7JPP2 + TEST_DATA_ITEM_12F6LZY6 + TEST_DATA_ITEM_12FCPQP9 + TEST_DATA_ITEM_12MBDC31 + TEST_DATA_ITEM_12MBFPV9 + TEST_DATA_ITEM_12PPWX76 + TEST_DATA_ITEM_12QFVS97 + TEST_DATA_ITEM_12QSDL49 + TEST_DATA_ITEM_12QVB1C9 + TEST_DATA_ITEM_12QX0K80 + TEST_DATA_ITEM_12R3SVM5 + TEST_DATA_ITEM_12THJBR1 + TEST_DATA_ITEM_12TKHH48 + TEST_DATA_ITEM_12V1MCV2 + TEST_DATA_ITEM_12V27BM9 + TEST_DATA_ITEM_12VBH621 + TEST_DATA_ITEM_12W99CF7 + TEST_DATA_ITEM_12WFRGG3 + TEST_DATA_ITEM_12Y71RB8 + TEST_DATA_ITEM_12YL4F48 + TEST_DATA_ITEM_134WCD50 + TEST_DATA_ITEM_135B2107 + TEST_DATA_ITEM_135W7XZ8 + TEST_DATA_ITEM_136KJJM5 + TEST_DATA_ITEM_13978WW3 + TEST_DATA_ITEM_139RY7J0 + TEST_DATA_ITEM_13CYT627 + TEST_DATA_ITEM_13GDWQH5 + TEST_DATA_ITEM_13KKBWZ4 + TEST_DATA_ITEM_13P11C13 + TEST_DATA_ITEM_13P1D8K8 + TEST_DATA_ITEM_13PL2VH9 + TEST_DATA_ITEM_13PRXVT9 + TEST_DATA_ITEM_13THLBR6 + TEST_DATA_ITEM_13ZGNYL5 + TEST_DATA_ITEM_145KKRP1 + TEST_DATA_ITEM_146YWG26 + TEST_DATA_ITEM_149QCWP2 + TEST_DATA_ITEM_14GJ8M40 + TEST_DATA_ITEM_14GJSJM5 + TEST_DATA_ITEM_14J211L8 + TEST_DATA_ITEM_14KFQZV4 + TEST_DATA_ITEM_14QJ4014 + TEST_DATA_ITEM_157C5S06 + TEST_DATA_ITEM_15H3RLY9 + TEST_DATA_ITEM_15LHYNV4 + TEST_DATA_ITEM_160DYBM0 + TEST_DATA_ITEM_16HRS1C1 + TEST_DATA_ITEM_1779WNW5 + TEST_DATA_ITEM_17BXPX26 + TEST_DATA_ITEM_17J18SJ0 + TEST_DATA_ITEM_17J1X165 + TEST_DATA_ITEM_17WSF072 + TEST_DATA_ITEM_17XFZWY7 + TEST_DATA_ITEM_19Y3V5C6 + TEST_DATA_ITEM_1B79XPY8 + + + + DATA.1 + + + DATA.2 + + + + + + + + 3 + + + \ No newline at end of file diff --git a/solr/modules/saved-search/src/test-files/monitor/dangling-test-single-doc-batch.json b/solr/modules/saved-search/src/test-files/monitor/dangling-test-single-doc-batch.json new file mode 100644 index 000000000000..19d5fff4a76c --- /dev/null +++ b/solr/modules/saved-search/src/test-files/monitor/dangling-test-single-doc-batch.json @@ -0,0 +1,10 @@ +{ + "params": { + "reverseSearchDocuments": [ + { + "id": "0", + "content1_s": "lift" + } + ] + } +} \ No newline at end of file diff --git a/solr/modules/saved-search/src/test-files/monitor/doc0.json b/solr/modules/saved-search/src/test-files/monitor/doc0.json new file mode 100644 index 000000000000..40ee08fd3eea --- /dev/null +++ b/solr/modules/saved-search/src/test-files/monitor/doc0.json @@ -0,0 +1,9 @@ +{ + "params": { + "reverseSearchDocuments": [ + { + "content_s": "elevator stairs" + } + ] + } +} \ No newline at end of file diff --git a/solr/modules/saved-search/src/test-files/monitor/doc1.json b/solr/modules/saved-search/src/test-files/monitor/doc1.json new file mode 100644 index 000000000000..af95e4a35a16 --- /dev/null +++ b/solr/modules/saved-search/src/test-files/monitor/doc1.json @@ -0,0 +1,9 @@ +{ + "params": { + "reverseSearchDocuments": [ + { + "content_s": "something else" + } + ] + } +} \ No newline at end of file diff --git a/solr/modules/saved-search/src/test-files/monitor/doc2.json b/solr/modules/saved-search/src/test-files/monitor/doc2.json new file mode 100644 index 000000000000..386b3bb68ad5 --- /dev/null +++ b/solr/modules/saved-search/src/test-files/monitor/doc2.json @@ -0,0 +1,10 @@ +{ + "params": { + "reverseSearchDocuments": [ + { + "content_s": "more weird content", + "other_content_s": "solr is cool" + } + ] + } +} \ No newline at end of file diff --git a/solr/modules/saved-search/src/test-files/monitor/elevator-doc.json b/solr/modules/saved-search/src/test-files/monitor/elevator-doc.json new file mode 100644 index 000000000000..83dcec61e805 --- /dev/null +++ b/solr/modules/saved-search/src/test-files/monitor/elevator-doc.json @@ -0,0 +1,12 @@ +{ + "params": { + "reverseSearchDocuments": [ + { + "id": "0", + "content0_s": "elevator stairs", + "content1_s": "winda schody", + "content2_s": "aufzug stufen" + } + ] + } +} \ No newline at end of file diff --git a/solr/modules/saved-search/src/test-files/monitor/multi-value-doc.json b/solr/modules/saved-search/src/test-files/monitor/multi-value-doc.json new file mode 100644 index 000000000000..8984517c33a0 --- /dev/null +++ b/solr/modules/saved-search/src/test-files/monitor/multi-value-doc.json @@ -0,0 +1,11 @@ +{ + "params": { + "reverseSearchDocuments": [ + { + "id": "4", + "content_s": ["elevator stairs", "something else", "test test test"], + "other_content_s": "I'm tired" + } + ] + } +} \ No newline at end of file diff --git a/solr/modules/saved-search/src/test-files/solr/collection1/core.properties b/solr/modules/saved-search/src/test-files/solr/collection1/core.properties new file mode 100644 index 000000000000..b0ef0785a797 --- /dev/null +++ b/solr/modules/saved-search/src/test-files/solr/collection1/core.properties @@ -0,0 +1,20 @@ +# +# /* +# * Licensed to the Apache Software Foundation (ASF) under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The ASF licenses this file to You under the Apache License, Version 2.0 +# * (the "License"); you may not use this file except in compliance with +# * the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +name=simple \ No newline at end of file diff --git a/solr/modules/saved-search/src/test-files/solr/collection1/schema-aliasing.xml b/solr/modules/saved-search/src/test-files/solr/collection1/schema-aliasing.xml new file mode 100644 index 000000000000..a7fec147ff1b --- /dev/null +++ b/solr/modules/saved-search/src/test-files/solr/collection1/schema-aliasing.xml @@ -0,0 +1,99 @@ + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/solr/modules/saved-search/src/test-files/solr/collection1/schema-stored-mq.xml b/solr/modules/saved-search/src/test-files/solr/collection1/schema-stored-mq.xml new file mode 100644 index 000000000000..2cd3d925a5f1 --- /dev/null +++ b/solr/modules/saved-search/src/test-files/solr/collection1/schema-stored-mq.xml @@ -0,0 +1,95 @@ + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/solr/modules/saved-search/src/test-files/solr/collection1/schema.xml b/solr/modules/saved-search/src/test-files/solr/collection1/schema.xml new file mode 100644 index 000000000000..49e9135e5e11 --- /dev/null +++ b/solr/modules/saved-search/src/test-files/solr/collection1/schema.xml @@ -0,0 +1,96 @@ + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/solr/modules/saved-search/src/test-files/solr/collection1/solrconfig-no-cache.xml b/solr/modules/saved-search/src/test-files/solr/collection1/solrconfig-no-cache.xml new file mode 100644 index 000000000000..d9f7389e65fe --- /dev/null +++ b/solr/modules/saved-search/src/test-files/solr/collection1/solrconfig-no-cache.xml @@ -0,0 +1,139 @@ + + + + + ${tests.luceneMatchVersion:LATEST} + + ${solr.data.dir:} + + + + + + + ${solr.lock.type:native} + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + ${solr.autoCommit.maxTime:15000} + false + + + ${solr.autoSoftCommit.maxTime:1000} + + + + 1024 + + + + + true + 20 + 200 + + + + + + + + + false + + + + + + + reverseSearch + + + + + explicit + json + true + id + + + + + explicit + json + true + id + + + + + 0 + false + xml + _version_ + + + + + 0 + /pingQuery + + + all + + + + + TermFilteredPresearcher + true + + + + + + + + + + text/plain; charset=UTF-8 + + + diff --git a/solr/modules/saved-search/src/test-files/solr/collection1/solrconfig-single-core.xml b/solr/modules/saved-search/src/test-files/solr/collection1/solrconfig-single-core.xml new file mode 100644 index 000000000000..5fc8f356f56b --- /dev/null +++ b/solr/modules/saved-search/src/test-files/solr/collection1/solrconfig-single-core.xml @@ -0,0 +1,161 @@ + + + + + ${tests.luceneMatchVersion:LATEST} + + ${solr.data.dir:} + + + + + + + ${solr.lock.type:native} + + 16 + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + ${solr.autoCommit.maxTime:15000} + false + + + ${solr.autoSoftCommit.maxTime:1000} + + + + 1024 + + + + + + true + 20 + 200 + + + + + + + + + false + + + + + + + + explicit + json + true + id + + + reverseSearch + facet + facet_module + highlight + stats + debug + expand + terms + + + + + explicit + json + true + id + + + + + 0 + false + xml + _version_ + + + + + 0 + /pingQuery + + + all + + + + + mySolrMonitorCache + MultipassTermFilteredPresearcher + true + 4 + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + diff --git a/solr/modules/saved-search/src/test-files/solr/collection1/solrconfig.xml b/solr/modules/saved-search/src/test-files/solr/collection1/solrconfig.xml new file mode 100644 index 000000000000..f5ef655170cd --- /dev/null +++ b/solr/modules/saved-search/src/test-files/solr/collection1/solrconfig.xml @@ -0,0 +1,140 @@ + + + + + ${tests.luceneMatchVersion:LATEST} + + ${solr.data.dir:} + + + + + + + ${solr.lock.type:native} + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + ${solr.autoCommit.maxTime:15000} + false + + + ${solr.autoSoftCommit.maxTime:1000} + + + + 1024 + + + + + + true + 20 + 200 + + + + + + + + + false + + + + + + + + explicit + json + true + id + + + + + explicit + json + true + id + + + + + 0 + false + xml + _version_ + + + + + 0 + /pingQuery + + + all + + + + + TermFilteredPresearcher + + + + + + + + + + text/plain; charset=UTF-8 + + + diff --git a/solr/modules/saved-search/src/test-files/solr/solr.xml b/solr/modules/saved-search/src/test-files/solr/solr.xml new file mode 100644 index 000000000000..af58afdeb320 --- /dev/null +++ b/solr/modules/saved-search/src/test-files/solr/solr.xml @@ -0,0 +1,28 @@ + + + + + + cores/ + + + ${urlScheme:} + + \ No newline at end of file diff --git a/solr/modules/saved-search/src/test/org/apache/solr/savedsearch/NoCacheSavedSearchTest.java b/solr/modules/saved-search/src/test/org/apache/solr/savedsearch/NoCacheSavedSearchTest.java new file mode 100644 index 000000000000..bebb3030cc2e --- /dev/null +++ b/solr/modules/saved-search/src/test/org/apache/solr/savedsearch/NoCacheSavedSearchTest.java @@ -0,0 +1,31 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.solr.savedsearch; + +import org.junit.BeforeClass; + +public class NoCacheSavedSearchTest extends SavedSearchTest { + + @BeforeClass + public static void beforeSuperClass() { + configString = "solrconfig-no-cache.xml"; + schemaString = "schema-aliasing.xml"; + } +} diff --git a/solr/modules/saved-search/src/test/org/apache/solr/savedsearch/SavedSearchTest.java b/solr/modules/saved-search/src/test/org/apache/solr/savedsearch/SavedSearchTest.java new file mode 100644 index 000000000000..25c9a38fb20f --- /dev/null +++ b/solr/modules/saved-search/src/test/org/apache/solr/savedsearch/SavedSearchTest.java @@ -0,0 +1,397 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.solr.savedsearch; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import org.apache.solr.BaseDistributedSearchTestCase; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.params.CommonParams; +import org.junit.Test; + +@SuppressWarnings("unchecked") +public class SavedSearchTest extends BaseDistributedSearchTestCase { + + @Test + public void testMonitorQuery() throws Exception { + index( + id, + Integer.toString(0), + SavedSearchDataValues.MONITOR_QUERY, + "content_s:\"elevator music\""); + index( + id, + Integer.toString(1), + SavedSearchDataValues.MONITOR_QUERY, + "{!xmlparser}steep stairs"); + index( + id, + Integer.toString(2), + SavedSearchDataValues.MONITOR_QUERY, + "content_s:\"elevator sounds\""); + index( + id, + Integer.toString(3), + SavedSearchDataValues.MONITOR_QUERY, + "content_s:\"elevator drop\""); + index( + id, + Integer.toString(4), + SavedSearchDataValues.MONITOR_QUERY, + "content_s:\"elevator stairs\""); + index( + id, + Integer.toString(5), + SavedSearchDataValues.MONITOR_QUERY, + "other_content_s:\"solr is cool\""); + index( + id, + Integer.toString(6), + SavedSearchDataValues.MONITOR_QUERY, + "other_content_s:\"solr is lame\""); + index(id, Integer.toString(7), SavedSearchDataValues.MONITOR_QUERY, "content_s:something"); + commit(); + handle.clear(); + handle.put("responseHeader", SKIP); + handle.put("response", SKIP); + + Object[] params = + new Object[] { + CommonParams.SORT, + SavedSearchDataValues.QUERY_ID + " desc", + CommonParams.JSON, + read("/monitor/doc1.json"), + CommonParams.QT, + "/reverseSearch" + }; + QueryResponse response = query(params); + System.out.println("Response = " + response); + validateDocList(response, List.of("7")); + } + + @Test + @ShardsFixed(num = 3) + public void testMonitorQueryWithUpdate() throws Exception { + index( + id, + Integer.toString(0), + SavedSearchDataValues.MONITOR_QUERY, + "content_s:\"elevator music\""); + index( + id, + Integer.toString(1), + SavedSearchDataValues.MONITOR_QUERY, + "{!xmlparser}steep stairs"); + index( + id, + Integer.toString(2), + SavedSearchDataValues.MONITOR_QUERY, + "content_s:\"elevator sounds\""); + index( + id, + Integer.toString(3), + SavedSearchDataValues.MONITOR_QUERY, + "content_s:\"elevator drop\""); + index( + id, + Integer.toString(4), + SavedSearchDataValues.MONITOR_QUERY, + "content_s:\"elevator stairs\""); + index( + id, + Integer.toString(5), + SavedSearchDataValues.MONITOR_QUERY, + "other_content_s:\"solr is cool\""); + index( + id, + Integer.toString(6), + SavedSearchDataValues.MONITOR_QUERY, + "other_content_s:\"solr is lame\""); + index(id, Integer.toString(7), SavedSearchDataValues.MONITOR_QUERY, "content_s:something"); + commit(); + handle.clear(); + handle.put("responseHeader", SKIP); + handle.put("response", SKIP); + + Object[] params = + new Object[] { + CommonParams.SORT, + SavedSearchDataValues.QUERY_ID + " desc", + CommonParams.JSON, + read("/monitor/doc0.json"), + CommonParams.QT, + "/reverseSearch" + }; + QueryResponse response = query(params); + System.out.println("Response = " + response); + validateDocList(response, List.of("4", "1")); + assertEquals(2, ((SolrDocumentList) response.getResponse().get("response")).size()); + + index( + id, + Integer.toString(5), + SavedSearchDataValues.MONITOR_QUERY, + "other_content_s:\"solr is lame\""); + index( + id, + Integer.toString(6), + SavedSearchDataValues.MONITOR_QUERY, + "other_content_s:\"solr is cool\""); + index( + id, + Integer.toString(1), + SavedSearchDataValues.MONITOR_QUERY, + "{!xmlparser}steep hill"); + index(id, Integer.toString(2), SavedSearchDataValues.MONITOR_QUERY, "content_s:elevator"); + commit(); + handle.clear(); + handle.put("responseHeader", SKIP); + handle.put("response", SKIP); + + params = + new Object[] { + CommonParams.SORT, + id + " desc", + CommonParams.JSON, + read("/monitor/doc2.json"), + CommonParams.QT, + "/reverseSearch" + }; + response = query(params); + validateDocList(response, List.of("6")); + } + + @Test + public void testDefaultParser() throws Exception { + index( + id, + Integer.toString(0), + SavedSearchDataValues.MONITOR_QUERY, + "content_s:\"elevator stairs\""); + index( + id, + Integer.toString(1), + SavedSearchDataValues.MONITOR_QUERY, + "content_s:\"something else\""); + commit(); + handle.clear(); + handle.put("responseHeader", SKIP); + handle.put("response", SKIP); + + Object[] params = + new Object[] { + CommonParams.SORT, + id + " desc", + CommonParams.JSON, + read("/monitor/multi-value-doc.json"), + CommonParams.QT, + "/reverseSearch" + }; + + QueryResponse response = query(params); + System.out.println("Response = " + response); + validateDocList(response, List.of("1", "0")); + } + + @Test + @ShardsFixed(num = 2) + public void testDisjunctionQuery() throws Exception { + index( + id, + Integer.toString(0), + SavedSearchDataValues.MONITOR_QUERY, + "{!xmlparser}elevatorwindastufen"); + index( + id, + Integer.toString(1), + SavedSearchDataValues.MONITOR_QUERY, + "{!xmlparser}nottheseterms"); + commit(); + handle.clear(); + handle.put("responseHeader", SKIP); + handle.put("response", SKIP); + + Object[] params = + new Object[] { + CommonParams.SORT, + id + " desc", + CommonParams.JSON, + read("/monitor/elevator-doc.json"), + CommonParams.QT, + "/reverseSearch" + }; + + QueryResponse response = query(params); + System.out.println("Response = " + response); + validateDocList(response, List.of("0")); + } + + @Test + public void testNoDanglingDecomposition() throws Exception { + index( + id, + Integer.toString(0), + SavedSearchDataValues.MONITOR_QUERY, + "{!xmlparser}elevatorlift"); + commit(); + handle.clear(); + handle.put("responseHeader", SKIP); + handle.put("response", SKIP); + + Object[] params = + new Object[] { + CommonParams.SORT, + id + " desc", + CommonParams.JSON, + read("/monitor/dangling-test-single-doc-batch.json"), + CommonParams.QT, + "/reverseSearch" + }; + + QueryResponse response = query(params); + System.out.println("Response = " + response); + validateDocList(response, List.of("0")); + + index( + id, + Integer.toString(0), + SavedSearchDataValues.MONITOR_QUERY, + "{!xmlparser}elevator"); + commit(); + response = query(params); + System.out.println("Response = " + response); + validateDocList(response, List.of()); + } + + @Test + public void testNotQuery() throws Exception { + index( + id, + Integer.toString(0), + SavedSearchDataValues.MONITOR_QUERY, + "*:* -content0_s:\"elevator stairs\""); + index( + id, + Integer.toString(1), + SavedSearchDataValues.MONITOR_QUERY, + "*:* -content_s:\"candy canes\""); + commit(); + handle.clear(); + handle.put("responseHeader", SKIP); + handle.put("response", SKIP); + + Object[] params = + new Object[] { + CommonParams.SORT, + id + " desc", + CommonParams.JSON, + read("/monitor/elevator-doc.json"), + CommonParams.QT, + "/reverseSearch" + }; + + QueryResponse response = query(params); + System.out.println("Response = " + response); + validateDocList(response, List.of("1")); + } + + @Test + public void testWildCardQuery() throws Exception { + index(id, Integer.toString(0), SavedSearchDataValues.MONITOR_QUERY, "content_s:te*"); + index(id, Integer.toString(1), SavedSearchDataValues.MONITOR_QUERY, "content_s:tes*"); + index(id, Integer.toString(2), SavedSearchDataValues.MONITOR_QUERY, "content_s:test*"); + index(id, Integer.toString(3), SavedSearchDataValues.MONITOR_QUERY, "content_s:tex*"); + index(id, Integer.toString(4), SavedSearchDataValues.MONITOR_QUERY, "content_s:tests*"); + commit(); + handle.clear(); + handle.put("responseHeader", SKIP); + handle.put("response", SKIP); + + Object[] params = + new Object[] { + CommonParams.SORT, + id + " desc", + CommonParams.JSON, + read("/monitor/multi-value-doc.json"), + CommonParams.QT, + "/reverseSearch" + }; + + QueryResponse response = query(params); + System.out.println("Response = " + response); + validateDocList(response, List.of("2", "1", "0")); + } + + @Test + public void testDeleteByQueryId() throws Exception { + index( + id, + Integer.toString(0), + SavedSearchDataValues.MONITOR_QUERY, + "{!xmlparser}elevatorlift"); + commit(); + handle.clear(); + handle.put("responseHeader", SKIP); + handle.put("response", SKIP); + + Object[] params = + new Object[] { + CommonParams.SORT, + id + " desc", + CommonParams.JSON, + read("/monitor/dangling-test-single-doc-batch.json"), + CommonParams.QT, + "/reverseSearch" + }; + + QueryResponse response = query(params); + validateDocList(response, List.of("0")); + + del(SavedSearchDataValues.QUERY_ID + ":0"); + commit(); + response = query(CommonParams.Q, "*:*"); + validateDocList(response, List.of()); + } + + void validateDocList(QueryResponse response, Object expectedValue) { + SolrDocumentList actualValues = (SolrDocumentList) response.getResponse().get("response"); + // minimal checks only so far; TODO: make this more comprehensive + if (expectedValue instanceof List) { + List expectedValues = (List) expectedValue; + int i = 0; + assertEquals(expectedValues.size(), actualValues.size()); + for (var ev : expectedValues) { + assertEquals(ev, actualValues.get(i).getFieldValue(SavedSearchDataValues.QUERY_ID)); + i++; + } + } + } + + protected String read(String resourceName) throws IOException, URISyntaxException { + final URL url = getClass().getResource(resourceName); + return Files.readString(Path.of(url.toURI()), StandardCharsets.UTF_8); + } +} diff --git a/solr/modules/saved-search/src/test/org/apache/solr/savedsearch/SingleCoreSavedSearchTest.java b/solr/modules/saved-search/src/test/org/apache/solr/savedsearch/SingleCoreSavedSearchTest.java new file mode 100644 index 000000000000..968ac60260d9 --- /dev/null +++ b/solr/modules/saved-search/src/test/org/apache/solr/savedsearch/SingleCoreSavedSearchTest.java @@ -0,0 +1,223 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.solr.savedsearch; + +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Locale; +import java.util.stream.IntStream; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CommonParams; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class SingleCoreSavedSearchTest extends SolrTestCaseJ4 { + + private final String monitorChain = "monitor"; + + @BeforeClass + public static void beforeTests() throws Exception { + initCore("solrconfig-single-core.xml", "schema-aliasing.xml"); + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + clearIndex(); + assertU(commit()); + } + + @Test + public void coexistWithRegularDocumentsTest() throws Exception { + String regularChain = "regular-ole-document"; + addDoc(adoc("id", "0", "content_s", "some unremarkable content"), regularChain); + addDoc(commit(), regularChain); + addDoc(adoc("id", "1", SavedSearchDataValues.MONITOR_QUERY, "content_s:test"), monitorChain); + addDoc(commit(), monitorChain); + URL url = getClass().getResource("/monitor/multi-value-doc.json"); + String json = Files.readString(Path.of(url.toURI()), StandardCharsets.UTF_8); + + String[] params = + new String[] { + CommonParams.SORT, "id desc", CommonParams.JSON, json, CommonParams.QT, "/reverseSearch" + }; + + assertQ(req(params), "//*[@numFound='1']"); + } + + @Test + public void queryStringIdTest() throws Exception { + addDoc(adoc("id", "1", SavedSearchDataValues.MONITOR_QUERY, "id:4"), monitorChain); + addDoc(adoc("id", "2", SavedSearchDataValues.MONITOR_QUERY, "id:4"), monitorChain); + addDoc(commit(), monitorChain); + URL url = getClass().getResource("/monitor/multi-value-doc.json"); + String json = Files.readString(Path.of(url.toURI()), StandardCharsets.UTF_8); + + String[] params = + new String[] { + CommonParams.SORT, "id desc", CommonParams.JSON, json, CommonParams.QT, "/reverseSearch" + }; + + assertQ(req(params), "//*[@numFound='2']"); + } + + @Test + public void missingQueryFieldTest() { + var ex = assertThrows(SolrException.class, () -> addDoc(adoc("id", "1"), monitorChain)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code()); + String lowerCaseMessage = ex.getMessage().toLowerCase(Locale.ROOT); + assertTrue(lowerCaseMessage.contains("missing")); + assertTrue(lowerCaseMessage.contains("mandatory")); + assertTrue(ex.getMessage().contains(SavedSearchDataValues.MONITOR_QUERY)); + assertTrue(lowerCaseMessage.contains("field")); + } + + @Test + public void multiPassPresearcherTest() throws Exception { + addDoc( + adoc( + "id", + "0", + SavedSearchDataValues.MONITOR_QUERY, + "content0_s:\"elevator stairs and escalator\""), + monitorChain); + addDoc( + adoc("id", "1", SavedSearchDataValues.MONITOR_QUERY, "content0_s:\"elevator test\""), + monitorChain); + addDoc( + adoc("id", "2", SavedSearchDataValues.MONITOR_QUERY, "content0_s:\"stairs test\""), + monitorChain); + addDoc( + adoc("id", "3", SavedSearchDataValues.MONITOR_QUERY, "content0_s:\"elevator stairs\""), + monitorChain); + addDoc(commit(), monitorChain); + URL url = getClass().getResource("/monitor/elevator-doc.json"); + String json = Files.readString(Path.of(url.toURI()), StandardCharsets.UTF_8); + + String[] params = + new String[] { + CommonParams.SORT, + "id desc", + CommonParams.JSON, + json, + CommonParams.QT, + "/select", + CommonParams.DEBUG_QUERY, + "true" + }; + + assertQ( + req(params), + "//*[@numFound='1']", + "/response/lst[@name=\"debug\"]/lst[@name=\"reverse-search-debug\"]/int[@name=\"queriesRun\"]/text()='1'"); + } + + @Test + public void manySegmentsQuery() throws Exception { + int count = 10_000; + IntStream.range(0, count) + .forEach( + i -> { + try { + addDoc( + adoc( + "id", + Integer.toString(i), + SavedSearchDataValues.MONITOR_QUERY, + "content_s:\"elevator stairs\""), + monitorChain); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }); + addDoc(commit(), monitorChain); + URL url = getClass().getResource("/monitor/multi-value-doc.json"); + String json = Files.readString(Path.of(url.toURI()), StandardCharsets.UTF_8); + + String[] params = + new String[] { + CommonParams.SORT, + "id desc", + CommonParams.JSON, + json, + CommonParams.QT, + "/reverseSearch", + CommonParams.DEBUG_QUERY, + "true", + "multiThreaded", + "true" + }; + + assertQ( + req(params), + "//*[@numFound='" + count + "']", + "/response/lst[@name=\"debug\"]/lst[@name=\"reverse-search-debug\"]/int[@name=\"queriesRun\"]/text()='" + + count + + "'"); + + IntStream.range(count / 2, count) + .forEach( + i -> { + try { + addDoc( + adoc( + "id", + Integer.toString(i), + SavedSearchDataValues.MONITOR_QUERY, + "content_s:\"x y\""), + monitorChain); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }); + addDoc(commit(), monitorChain); + assertQ( + req(params), + "//*[@numFound='" + count / 2 + "']", + "/response/lst[@name=\"debug\"]/lst[@name=\"reverse-search-debug\"]/int[@name=\"queriesRun\"]/text()='" + + count / 2 + + "'"); + } + + @Test + public void fieldExistsQuery() throws Exception { + // FieldExistsQuery checks if the ephemeral single-doc index is open. + // This was a bug that flew under the radar because not a lot of places in + // Lucene check if the LeafReader is still open. + addDoc( + adoc("id", String.valueOf(1), SavedSearchDataValues.MONITOR_QUERY, "id:[* TO *]"), + monitorChain); + addDoc(commit(), monitorChain); + URL url = getClass().getResource("/monitor/multi-value-doc.json"); + String json = Files.readString(Path.of(url.toURI()), StandardCharsets.UTF_8); + + String[] params = + new String[] { + CommonParams.SORT, "id desc", CommonParams.JSON, json, CommonParams.QT, "/reverseSearch" + }; + + assertQ(req(params), "//*[@numFound='1']"); + } +} diff --git a/solr/modules/saved-search/src/test/org/apache/solr/savedsearch/StoredSavedSearchTest.java b/solr/modules/saved-search/src/test/org/apache/solr/savedsearch/StoredSavedSearchTest.java new file mode 100644 index 000000000000..fadde0447186 --- /dev/null +++ b/solr/modules/saved-search/src/test/org/apache/solr/savedsearch/StoredSavedSearchTest.java @@ -0,0 +1,41 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.solr.savedsearch; + +import org.junit.BeforeClass; +import org.junit.Test; + +public class StoredSavedSearchTest extends SavedSearchTest { + + @BeforeClass + public static void beforeSuperClass() { + schemaString = "schema-stored-mq.xml"; + } + + @Test + public void indexBigQueryTest() throws Exception { + index( + id, + Integer.toString(0), + SavedSearchDataValues.MONITOR_QUERY, + "{!xmlparser}" + read("/monitor/BigDisjunctionQuery.xml")); + commit(); + } +}